diff --git a/CODEOWNERS b/CODEOWNERS index 75e43ef23..ef04bdf5b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -4,3 +4,4 @@ # Snaps /src/snaps/ @rbhanda +/src/sourceBuild/ @dotnet/source-build-internal \ No newline at end of file diff --git a/eng/Build.props b/eng/Build.props index 917aa8993..4df683bab 100644 --- a/eng/Build.props +++ b/eng/Build.props @@ -1,8 +1,12 @@ - + + + + + diff --git a/eng/SourceBuild.Version.Details.xml b/eng/SourceBuild.Version.Details.xml new file mode 100644 index 000000000..b11f1229a --- /dev/null +++ b/eng/SourceBuild.Version.Details.xml @@ -0,0 +1,28 @@ + + + + + https://github.com/mono/linker + f2588193553431636b9853b0f87209fa395a72c5 + + linker + + + + + https://github.com/dotnet/arcade + a3377cccde8639089f99107e2ba5df2c8cbe6394 + + + + https://github.com/dotnet/source-build-reference-packages + def2e2c6dc5064319250e2868a041a3dc07f9579 + + + + https://github.com/dotnet/sourcelink + 4b584dbc392bb1aad49c2eb1ab84d8b489b6dccc + + + + diff --git a/eng/Versions.props b/eng/Versions.props index 6826ba349..c43c69f40 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -151,6 +151,15 @@ 2.0.0 17.0.0-preview-20210518-01 + + + + 15.7.179 + 15.7.179 + 0.1.0-6.0.100-bootstrap.3 + 6.0.100-ci.main.806 diff --git a/src/SourceBuild/Arcade/README.md b/src/SourceBuild/Arcade/README.md new file mode 100644 index 000000000..914a87b3f --- /dev/null +++ b/src/SourceBuild/Arcade/README.md @@ -0,0 +1 @@ +The source and targets in the `src/SourceBuild/Arcade` directory are intended to move into the Arcade repo at some point. They are added here for ease of development while developing the tarball generation process. See https://github.com/dotnet/source-build/issues/2295 \ No newline at end of file diff --git a/src/SourceBuild/Arcade/src/SourceBuild.Tasks.csproj b/src/SourceBuild/Arcade/src/SourceBuild.Tasks.csproj new file mode 100644 index 000000000..735289e70 --- /dev/null +++ b/src/SourceBuild/Arcade/src/SourceBuild.Tasks.csproj @@ -0,0 +1,17 @@ + + + net5.0 + false + false + disable + + + + + + + + + + + diff --git a/src/SourceBuild/Arcade/src/Tarball_ReadSourceBuildIntermediateNupkgDependencies.cs b/src/SourceBuild/Arcade/src/Tarball_ReadSourceBuildIntermediateNupkgDependencies.cs new file mode 100644 index 000000000..fbd497b72 --- /dev/null +++ b/src/SourceBuild/Arcade/src/Tarball_ReadSourceBuildIntermediateNupkgDependencies.cs @@ -0,0 +1,135 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System.Collections.Generic; +using System.Linq; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks +{ + /// + /// Reads entries in a Version.Details.xml file to find intermediate nupkg dependencies. For + /// each dependency with a "SourceBuild" element, adds an item to the "Dependencies" output. + /// + public class Tarball_ReadSourceBuildIntermediateNupkgDependencies : Task + { + [Required] + public string VersionDetailsXmlFile { get; set; } + + [Required] + public string SourceBuildIntermediateNupkgPrefix { get; set; } + + /// + /// The intermediate nupkg RID to use if any RID-specific intermediate nupkgs are required. + /// If this parameter isn't specified, RID-specific intermediate nupkgs can't be used and + /// this task fails. + /// + public string SourceBuildIntermediateNupkgRid { get; set; } + + /// + /// %(Identity): NuGet package ID. + /// %(Name): The Name of the dependency from Version.Details.xml. + /// %(ExactVersion): NuGet package version. This can be used to look up the restored package + /// contents in a package cache. + /// %(Version): NuGet package version, wrapped in "[version]" syntax for exact match. + /// %(Uri): The URI for the repo. + /// %(Sha): The commit Sha for the dependency. + /// %(SourceBuildRepoName): The repo name to use in source-build. + /// + [Output] + public ITaskItem[] Dependencies { get; set; } + + public override bool Execute() + { + XElement root = XElement.Load(VersionDetailsXmlFile, LoadOptions.PreserveWhitespace); + + XName CreateQualifiedName(string plainName) + { + return root.GetDefaultNamespace().GetName(plainName); + } + + Dependencies = root + .Elements() + .Elements(CreateQualifiedName("Dependency")) + .Select(d => + { + XElement sourceBuildElement = d.Element(CreateQualifiedName("SourceBuild")); + + if (sourceBuildElement == null) + { + // Ignore element: doesn't represent a source-build dependency. + return null; + } + + string repoName = sourceBuildElement.Attribute("RepoName")?.Value; + + if (string.IsNullOrEmpty(repoName)) + { + Log.LogError($"Dependency SourceBuild RepoName null or empty in '{VersionDetailsXmlFile}' element {d}"); + return null; + } + + string dependencyName = d.Attribute("Name")?.Value ?? string.Empty; + + if (string.IsNullOrEmpty(dependencyName)) + { + // Log name missing as FYI, but this is not an error case for source-build. + Log.LogMessage($"Dependency Name null or empty in '{VersionDetailsXmlFile}' element {d}"); + } + + string dependencyVersion = d.Attribute("Version")?.Value; + + string uri = d.Element(CreateQualifiedName("Uri"))?.Value; + string sha = d.Element(CreateQualifiedName("Sha"))?.Value; + string sourceBuildRepoName = sourceBuildElement.Attribute("RepoName")?.Value; + + if (string.IsNullOrEmpty(dependencyVersion)) + { + // We need a version to bring down an intermediate nupkg. Fail. + Log.LogError($"Dependency Version null or empty in '{VersionDetailsXmlFile}' element {d}"); + return null; + } + + string identity = SourceBuildIntermediateNupkgPrefix + repoName; + + bool.TryParse( + sourceBuildElement.Attribute("ManagedOnly")?.Value, + out bool managedOnly); + + // If RID-specific, add the RID to the end of the identity. + if (!managedOnly) + { + if (string.IsNullOrEmpty(SourceBuildIntermediateNupkgRid)) + { + Log.LogError( + $"Parameter {nameof(SourceBuildIntermediateNupkgRid)} was " + + "not specified, indicating this project depends only on managed " + + "inputs. However, source-build element is not ManagedOnly: " + + sourceBuildElement); + return null; + } + + identity += "." + SourceBuildIntermediateNupkgRid; + } + + return new TaskItem( + identity, + new Dictionary + { + ["Name"] = dependencyName, + ["Version"] = $"[{dependencyVersion}]", + ["ExactVersion"] = dependencyVersion, + ["Uri"] = uri, + ["Sha"] = sha, + ["SourceBuildRepoName"] = sourceBuildRepoName + }); + }) + .Where(d => d != null) + .ToArray(); + + return !Log.HasLoggedErrors; + } + } +} diff --git a/src/SourceBuild/Arcade/src/Tarball_WriteSourceRepoProperties.cs b/src/SourceBuild/Arcade/src/Tarball_WriteSourceRepoProperties.cs new file mode 100644 index 000000000..58dbab586 --- /dev/null +++ b/src/SourceBuild/Arcade/src/Tarball_WriteSourceRepoProperties.cs @@ -0,0 +1,204 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Tasks; +using Microsoft.Build.Utilities; +using NuGet.Versioning; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml.Serialization; + +namespace Microsoft.DotNet.SourceBuild.Tasks +{ + /// + /// Writes a props file to the given directory for each dependency specified + /// plus adds or updates an existing props file with all dependencies. The + /// intention is for the props file to be included by a source-build build + /// to get metadata about each dependent repo. + /// + public class Tarball_WriteSourceRepoProperties : Task + { + /// + /// The directory to write the props files to. + /// + [Required] + public string SourceBuildMetadataDir { get; set; } + + /// + /// Dependencies to include in the props files. + /// + /// %(Identity): NuGet package ID. + /// %(Name): The Name of the dependency from Version.Details.xml. + /// %(ExactVersion): NuGet package version. This can be used to look up the restored package + /// contents in a package cache. + /// %(Version): NuGet package version, wrapped in "[version]" syntax for exact match. + /// %(Uri): The URI for the repo. + /// %(Sha): The commit Sha for the dependency. + /// %(SourceBuildRepoName): The repo name to use in source-build. + /// + /// + [Required] + public ITaskItem[] Dependencies { get; set; } + + public override bool Execute() + { + var allRepoProps = new Dictionary(); + + foreach (var dependency in Dependencies.Select(dep => + new { + Name = dep.GetMetadata("Name"), + Version = dep.GetMetadata("ExactVersion"), + Sha = dep.GetMetadata("Sha"), + Uri = dep.GetMetadata("Uri") + })) + { + string repoName = GetDefaultRepoNameFromUrl(dependency.Uri); + string safeRepoName = repoName.Replace("-", ""); + string propsPath = Path.Combine(SourceBuildMetadataDir, $"{repoName}.props"); + DerivedVersion derivedVersion = GetVersionInfo(dependency.Version, "0"); + var repoProps = new Dictionary + { + ["GitCommitHash"] = dependency.Sha, + ["OfficialBuildId"] = derivedVersion.OfficialBuildId, + ["OutputPackageVersion"] = dependency.Version, + ["PreReleaseVersionLabel"] = derivedVersion.PreReleaseVersionLabel, + ["IsStable"] = string.IsNullOrWhiteSpace(derivedVersion.PreReleaseVersionLabel) ? "true" : "false", + }; + WritePropsFile(propsPath, repoProps); + allRepoProps[$"{safeRepoName}GitCommitHash"] = dependency.Sha; + allRepoProps[$"{safeRepoName}OutputPackageVersion"] = dependency.Version; + } + string allRepoPropsPath = Path.Combine(SourceBuildMetadataDir, "AllRepoVersions.props"); + Log.LogMessage(MessageImportance.Normal, $"[{DateTimeOffset.Now}] Writing all repo versions to {allRepoPropsPath}"); + UpdatePropsFile(allRepoPropsPath, allRepoProps); + + return !Log.HasLoggedErrors; + } + + /// + /// Reverse a version in the Arcade style (https://github.com/dotnet/arcade/blob/fb92b14d8cd07cf44f8f7eefa8ac58d7ffd05f3f/src/Microsoft.DotNet.Arcade.Sdk/tools/Version.BeforeCommonTargets.targets#L18) + /// back to an OfficialBuildId + ReleaseLabel which we can then supply to get the same resulting version number. + /// + /// The complete version, e.g. 1.0.0-beta1-19720.5 + /// The current commit count of the repo. This is used for some repos that do not use the standard versioning scheme. + /// + private static DerivedVersion GetVersionInfo(string version, string commitCount) + { + var nugetVersion = new NuGetVersion(version); + + if (!string.IsNullOrWhiteSpace(nugetVersion.Release)) + { + var releaseParts = nugetVersion.Release.Split('-', '.'); + if (releaseParts.Length == 2) + { + if (releaseParts[1].TrimStart('0') == commitCount) + { + // core-sdk does this - OfficialBuildId is only used for their fake package and not in anything shipped + return new DerivedVersion { OfficialBuildId = DateTime.Now.ToString("yyyyMMdd.1"), PreReleaseVersionLabel = releaseParts[0] }; + } + else + { + // NuGet does this - arbitrary build IDs + return new DerivedVersion { OfficialBuildId = releaseParts[1], PreReleaseVersionLabel = releaseParts[0] }; + } + } + else if (releaseParts.Length == 3) + { + // VSTest uses full dates for the first part of their preview build numbers + if (DateTime.TryParseExact(releaseParts[1], "yyyyMMdd", new CultureInfo("en-US"), DateTimeStyles.AssumeLocal, out DateTime fullDate)) + { + return new DerivedVersion { OfficialBuildId = $"{releaseParts[1]}.{releaseParts[2]}", PreReleaseVersionLabel = releaseParts[0] }; + } + else if (int.TryParse(releaseParts[1], out int datePart) && int.TryParse(releaseParts[2], out int buildPart)) + { + if (datePart > 1 && datePart < 8 && buildPart > 1000 && buildPart < 10000) + { + return new DerivedVersion { OfficialBuildId = releaseParts[2], PreReleaseVersionLabel = $"{releaseParts[0]}.{releaseParts[1]}" }; + } + else + { + return new DerivedVersion { OfficialBuildId = $"20{((datePart / 1000))}{((datePart % 1000) / 50):D2}{(datePart % 50):D2}.{buildPart}", PreReleaseVersionLabel = releaseParts[0] }; + } + } + } + else if (releaseParts.Length == 4) + { + // new preview version style, e.g. 5.0.0-preview.7.20365.12 + if (int.TryParse(releaseParts[2], out int datePart) && int.TryParse(releaseParts[3], out int buildPart)) + { + return new DerivedVersion { OfficialBuildId = $"20{((datePart / 1000))}{((datePart % 1000) / 50):D2}{(datePart % 50):D2}.{buildPart}", PreReleaseVersionLabel = $"{releaseParts[0]}.{releaseParts[1]}" }; + } + } + } + else + { + // finalized version number (x.y.z) - probably not our code + // VSTest, Application Insights, Newtonsoft.Json do this + return new DerivedVersion { OfficialBuildId = DateTime.Now.ToString("yyyyMMdd.1"), PreReleaseVersionLabel = string.Empty }; + } + + throw new FormatException($"Can't derive a build ID from version {version} (commit count {commitCount}, release {string.Join(";", nugetVersion.Release.Split('-', '.'))})"); + } + + private static string GetDefaultRepoNameFromUrl(string repoUrl) + { + if (repoUrl.EndsWith(".git")) + { + repoUrl = repoUrl.Substring(0, repoUrl.Length - ".git".Length); + } + return repoUrl.Substring(repoUrl.LastIndexOf("/") + 1); + } + + private static void UpdatePropsFile(string filePath, Dictionary properties) + { + if (!File.Exists(filePath)) + { + WritePropsFile(filePath, properties); + } + else + { + var content = new StringBuilder(); + foreach (var line in File.ReadAllLines(filePath)) + { + content.AppendLine(line); + if (line.Contains("")) + { + foreach (var propName in properties.Keys.OrderBy(k => k)) + { + content.AppendLine($" <{propName}>{properties[propName]}"); + } + } + } + File.WriteAllText(filePath, content.ToString()); + } + } + + private static void WritePropsFile(string filePath, Dictionary properties) + { + var content = new StringBuilder(); + content.AppendLine(""); + content.AppendLine(""); + content.AppendLine(" "); + foreach (var propName in properties.Keys.OrderBy(k => k)) + { + content.AppendLine($" <{propName}>{properties[propName]}"); + } + content.AppendLine(" "); + content.AppendLine(""); + File.WriteAllText(filePath, content.ToString()); + } + + private class DerivedVersion + { + internal string OfficialBuildId { get; set; } + internal string PreReleaseVersionLabel { get; set; } + } + } +} diff --git a/src/SourceBuild/Arcade/tools/BuildTasks.props b/src/SourceBuild/Arcade/tools/BuildTasks.props new file mode 100644 index 000000000..4014b87ad --- /dev/null +++ b/src/SourceBuild/Arcade/tools/BuildTasks.props @@ -0,0 +1,7 @@ + + + + + $(RepoRoot)\artifacts\bin\SourceBuild.Tasks\$(Configuration)\net5.0\SourceBuild.Tasks.dll + + \ No newline at end of file diff --git a/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets b/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets new file mode 100644 index 000000000..914cf1ad3 --- /dev/null +++ b/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets @@ -0,0 +1,179 @@ + + + + + + + + + + + + + + $(RepoRoot)artifacts/tarball/ + $([MSBuild]::EnsureTrailingSlash('$(TarballDir)')) + $(TarballRootDir)src/ + $(TarballRootDir)git-info/ + quiet + + + + + + + + + + + + + + + + + + + + + + + + + + $(GitHubRepositoryName) + 1.0.0 + 1.0.0 + @(RootRepoCommitSha) + @(RootRepoUri) + $(GitHubRepositoryName) + + + + + + + %(SourceBuildRepos.SourceBuildRepoName) + %(SourceBuildRepos.Uri) + %(SourceBuildRepos.Sha) + + + + + + + + + + + + + $(SourceBuildRepoName).$(RepoSha)/ + $(TarballSourceDir)$(SourceDir) + $(TarballSourceDir)$(SourceDir)eng/ + $(TarballRepoSourceEngDir)Version.Details.xml + -q + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(TarballRootDir)packages/archive/ + Private.SourceBuilt.Artifacts + https://dotnetcli.azureedge.net/source-built-artifacts/assets/ + archiveArtifacts.txt + $(ExternalTarballsDir)$(ArchiveArtifactsTextFileName) + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/BuildSourceBuildTarball.proj b/src/SourceBuild/tarball/BuildSourceBuildTarball.proj new file mode 100644 index 000000000..147239862 --- /dev/null +++ b/src/SourceBuild/tarball/BuildSourceBuildTarball.proj @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/SourceBuild/tarball/content/Directory.Build.props b/src/SourceBuild/tarball/content/Directory.Build.props new file mode 100644 index 000000000..9c556df3f --- /dev/null +++ b/src/SourceBuild/tarball/content/Directory.Build.props @@ -0,0 +1,268 @@ + + + + false + <_SuppressSdkImports>true + Release + + + + + + netstandard2.0 + + false + + + + + $([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant()) + $(BuildArchitecture) + x64 + + true + + + false + false + true + false + + + + + false + + + + $(MSBuildThisFileDirectory) + $(ProjectDir)targets/ + $(ProjectDir)keys/ + $([MSBuild]::EnsureTrailingSlash('$(CustomDotNetSdkDir)')) + $([MSBuild]::NormalizeDirectory('$(DOTNET_INSTALL_DIR)')) + $(ProjectDir).dotnet/ + $(DotNetCliToolDir)dotnet + $(ProjectDir)patches/ + $(NuGetPackageRoot) + $(ProjectDir)packages/restored/ + $(PackagesDir)ArcadeBootstrapPackage/ + + $(NETCoreSdkVersion) + $(DotNetCliToolDir)sdk/$(SDK_VERSION)/ + $(DotNetSdkDir)SdkResolvers/ + $(DotNetCliToolDir)sdk/$(SDK_VERSION)/ + + + + + false + + + + minimal + + + + false + + + + .cmd + .sh + .zip + .tar.gz + + + + $(ProjectDir)src/ + $(ProjectDir).gitmodules + $(ProjectDir)ProdConFeed.txt + + + + + $(ProjectDir)artifacts/ + $(ProjectDir)eng/ + + + + $(ArtifactsDir) + $(BaseOutputPath)src/ + $(ProjectDir).git/modules/src/ + $(ProjectDir)Tools/ + $(ToolsDir)source-built/ + $(ProjectDir)tools-local/ + $(ToolsLocalDir)tasks/ + + $(TaskDirectory)Microsoft.DotNet.SourceBuild.Tasks.XPlat/bin/$(Configuration)/ + $(XPlatTasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.XPlat.dll + + $(TaskDirectory)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/ + $(LeakDetectionTasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.dll + + $(BaseOutputPath)obj/ + $(BaseOutputPath)$(Platform)/$(Configuration)/ + $(BaseIntermediatePath)$(Platform)/$(Configuration)/ + $(IntermediatePath)blobs/ + $(IntermediatePath)nuget-packages/ + $(IntermediatePath)blob-feed/ + $(SourceBuiltBlobFeedDir)packages/ + $(SourceBuiltBlobFeedDir)assets/ + $(ProjectDir)packages/prebuilt/ + $(ProjectDir)packages/previouslyRestored/ + $(ProjectDir)packages/source-built/ + $(CustomPrebuiltSourceBuiltPackagesPath)/ + $(OutputPath) + $(LocalBlobStorageRoot)Sdk/ + $(LocalBlobStorageRoot)Runtime/ + $(LocalBlobStorageRoot)aspnetcore/Runtime/ + $(IntermediatePath)RestoreSources.props + $(IntermediatePath)PackageVersions.props + $(IntermediatePath)GennedPackageVersions.props + $(IntermediatePath)PackageVersions.props + $(BaseOutputPath)logs/ + $(BaseOutputPath)msbuild-debug/ + $(BaseOutputPath)roslyn-debug/ + $(BaseOutputPath)aspnet-debug + $(AspNetRazorBuildServerLogDir)razor-build-server.log + + $(BaseOutputPath)git-info/ + + $(ProjectDir)git-info/ + $(GitInfoOutputDir)$(RepositoryName).props + $(GitInfoOutputDir)AllRepoVersions.props + $(GitInfoOfflineDir)$(RepositoryName).props + $(GitInfoOfflineDir)AllRepoVersions.props + $(BaseOutputPath)prebuilt-report/ + $(PackageReportDir)prebuilt-usage.xml + $(PackageReportDir)poison-usage.xml + $(PackageReportDir)poison-catalog.xml + .prebuilt.xml + $(PackageReportDir)poison-source-built-catalog.xml + .source-built.xml + $(PackageReportDir)all-project-assets-json-files.zip + $(PackageReportDir)prodcon-build.xml + $(PackageReportDir)poisoned.txt + $(BaseOutputPath)conflict-report/ + $(PackageReportDir)PrebuiltBurndownData-offline.csv + $(PackageReportDir)PrebuiltBurndownData-online.csv + $(IntermediatePath)reference-packages/ + $(IntermediatePath)text-only-packages/ + $(IntermediatePath)external-tarballs/ + + $(ProjectDir)packages/archive/ + $(ProjectDir)packages/reference/ + $(ProjectDir)packages/text-only/ + $(ReferencePackagesBaseDir)packages/ + Private.SourceBuilt.Artifacts + https://dotnetcli.azureedge.net/source-built-artifacts/assets/ + archiveArtifacts.txt + $(ExternalTarballsDir)$(ArchiveArtifactsTextFileName) + $(ToolsLocalDir)prebuilt-baseline- + $(BaselineDataFile)offline.xml + $(BaselineDataFile)online.xml + $(OfflineBaselineDataFile) + $(OnlineBaselineDataFile) + + $(ProjectDir)test/exclusions/ + + + + + + + ROOTFS_DIR=$(BaseIntermediatePath)crossrootfs/arm + ROOTFS_DIR=$(BaseIntermediatePath)crossrootfs/armel + + + + $([System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier) + + Windows_NT + OSX + Linux + FreeBSD + + + + freebsd-$(Platform) + osx-$(Platform) + linux-$(Platform) + win-$(Platform) + + + + + known-good + known-good-tests + + + + + $(BaseIntermediatePath)semaphores/ + + + + + source + 30000001-1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/Directory.Build.targets b/src/SourceBuild/tarball/content/Directory.Build.targets new file mode 100644 index 000000000..c9cd51f95 --- /dev/null +++ b/src/SourceBuild/tarball/content/Directory.Build.targets @@ -0,0 +1,63 @@ + + + + + + + + + + + + + + + + + + + + $(IgnoredRepos);https://dev.azure.com/dnceng/internal/_git/dotnet-optimization + $(IgnoredRepos);https://dev.azure.com/devdiv/DevDiv/_git/DotNet-Trusted + $(IgnoredRepos);https://devdiv.visualstudio.com/DevDiv/_git/DotNet-Trusted + $(IgnoredRepos);https://dnceng@dev.azure.com/dnceng/internal/_git/dotnet-optimization + $(IgnoredRepos);https://dev.azure.com/dnceng/internal/_git/dotnet-core-setup + $(IgnoredRepos);https://github.com/dotnet/source-build-reference-packages + + $(ClonedSubmoduleDirectory) + + $(DarcCloneArguments) --git-dir-folder $(ClonedSubmoduleGitRootDirectory) + $(DarcCloneArguments) --include-toolset + $(DarcCloneArguments) --ignore-repos "$(IgnoredRepos)" + $(DarcCloneArguments) --debug + + + bogus + $(DarcCloneArguments) --azdev-pat $(AzDoPat) + $(DarcCloneArguments) --github-pat bogus + + + $(DarcCloneArguments) --depth 0 + + $(DotNetCliToolDir)dotnet $(DarcDll) clone $(DarcCloneArguments) + + + + + + + + $([System.IO.File]::ReadAllText('$(ProdConFeedPath)').Trim()) + $(ProdConBlobFeedUrl.Replace('https://dotnetfeed.blob.core.windows.net/', '$(ProdConBlobFeedUrlPrefix)')) + + + + diff --git a/src/SourceBuild/tarball/content/build.proj b/src/SourceBuild/tarball/content/build.proj new file mode 100644 index 000000000..c8f1995ce --- /dev/null +++ b/src/SourceBuild/tarball/content/build.proj @@ -0,0 +1,197 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + N/A + + + + + + + + + + N/A + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ./smoke-test.sh + $(SmokeTestCommand) --minimal + $(SmokeTestCommand) --projectOutput + $(SmokeTestCommand) --configuration $(Configuration) + $(SmokeTestCommand) --archiveRestoredPackages + + + $(SmokeTestCommand) --excludeWebHttpsTests + + + + + + + + + + + + + + + + + + + $(RelativeBlobPath)/%(Filename)%(Extension) + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/build.sh b/src/SourceBuild/tarball/content/build.sh new file mode 100755 index 000000000..13f8ccc7b --- /dev/null +++ b/src/SourceBuild/tarball/content/build.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +usage() { + echo "usage: $0 [options]" + echo "options:" + echo " --with-ref-packages use the specified directory of reference packages" + echo " --with-packages use the specified directory of previously-built packages" + echo " --with-sdk use the SDK in the specified directory for bootstrapping" + echo "use -- to send the remaining arguments to MSBuild" + echo "" +} + +SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)" + +MSBUILD_ARGUMENTS=("/p:OfflineBuild=true" "/flp:v=detailed") +CUSTOM_REF_PACKAGES_DIR='' +CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR='' +alternateTarget=false +CUSTOM_SDK_DIR='' + +while :; do + if [ $# -le 0 ]; then + break + fi + + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + --run-smoke-test) + alternateTarget=true + MSBUILD_ARGUMENTS+=( "/t:RunSmokeTest" ) + ;; + --with-packages) + CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR="$(cd -P "$2" && pwd)" + if [ ! -d "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" ]; then + echo "Custom prviously built packages directory '$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR' does not exist" + exit 1 + fi + MSBUILD_ARGUMENTS+=( "/p:CustomPrebuiltSourceBuiltPackagesPath=$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" ) + shift + ;; + --with-sdk) + CUSTOM_SDK_DIR="$(cd -P "$2" && pwd)" + if [ ! -d "$CUSTOM_SDK_DIR" ]; then + echo "Custom SDK directory '$CUSTOM_SDK_DIR' does not exist" + exit 1 + fi + if [ ! -x "$CUSTOM_SDK_DIR/dotnet" ]; then + echo "Custom SDK '$CUSTOM_SDK_DIR/dotnet' does not exist or is not executable" + exit 1 + fi + shift + ;; + --) + shift + echo "Detected '--': passing remaining parameters '$@' as build.sh arguments." + break + ;; + -?|-h|--help) + usage + exit 0 + ;; + *) + echo "Unrecognized argument '$1'" + usage + exit 1 + ;; + esac + shift +done + +if [ -f "$SCRIPT_ROOT/packages/archive/archiveArtifacts.txt" ]; then + ARCHIVE_ERROR=0 + if [ ! -d "$SCRIPT_ROOT/.dotnet" ] && [ "$CUSTOM_SDK_DIR" == "" ]; then + echo "ERROR: SDK not found at $SCRIPT_ROOT/.dotnet" + ARCHIVE_ERROR=1 + fi + if [ ! -f $SCRIPT_ROOT/packages/archive/Private.SourceBuilt.Artifacts*.tar.gz ] && [ "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" == "" ]; then + echo "ERROR: Private.SourceBuilt.Artifacts artifact not found at $SCRIPT_ROOT/packages/archive/ - Either run prep.sh or pass --with-packages parameter" + ARCHIVE_ERROR=1 + fi + if [ $ARCHIVE_ERROR == 1 ]; then + echo "" + echo " Errors detected in tarball. To prep the tarball, run prep.sh while online to install an SDK" + echo " and Private.SourceBuilt.Artifacts tarball. After prepping the tarball, the tarball can be" + echo " built offline. As an alternative to prepping the tarball, these assets can be provided using" + echo " the --with-sdk and --with-packages parameters" + exit 1 + fi +fi + +if [ -d "$CUSTOM_SDK_DIR" ]; then + export SDK_VERSION=`"$CUSTOM_SDK_DIR/dotnet" --version` + export CLI_ROOT="$CUSTOM_SDK_DIR" + export _InitializeDotNetCli="$CLI_ROOT/dotnet" + export CustomDotNetSdkDir="$CLI_ROOT" + echo "Using custom bootstrap SDK from '$CLI_ROOT', version '$SDK_VERSION'" +else + sdkLine=`grep -m 1 'dotnet' "$SCRIPT_ROOT/global.json"` + sdkPattern="\"dotnet\" *: *\"(.*)\"" + if [[ $sdkLine =~ $sdkPattern ]]; then + export SDK_VERSION=${BASH_REMATCH[1]} + export CLI_ROOT="$SCRIPT_ROOT/.dotnet" + fi +fi + +packageVersionsPath='' +restoredPackagesDir="$SCRIPT_ROOT/packages/restored" + +if [ -d "$SCRIPT_ROOT/packages/archive" ]; then + sourceBuiltArchive=`find $SCRIPT_ROOT/packages/archive -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'` + if [ -f "$sourceBuiltArchive" ]; then + tar -xzf "$sourceBuiltArchive" -C /tmp PackageVersions.props + packageVersionsPath=/tmp/PackageVersions.props + fi +else + if [ -f "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR/PackageVersions.props" ]; then + packageVersionsPath="$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR/PackageVersions.props" + fi +fi + +if [ ! -f "$packageVersionsPath" ]; then + echo "Cannot find PackagesVersions.props. Debugging info:" + echo " Attempted archive path: $SCRIPT_ROOT/packages/archive" + echo " Attempted custom PVP path: $CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR/PackageVersions.props" + exit 1 +fi + +arcadeSdkLine=`grep -m 1 'MicrosoftDotNetArcadeSdkVersion' "$packageVersionsPath"` +versionPattern="(.*)" +if [[ $arcadeSdkLine =~ $versionPattern ]]; then + export ARCADE_BOOTSTRAP_VERSION=${BASH_REMATCH[1]} + + # Ensure that by default, the bootstrap version of the Arcade SDK is used. Source-build infra + # projects use bootstrap Arcade SDK, and would fail to find it in the tarball build. The repo + # projects overwrite this so that they use the source-built Arcade SDK instad. + export SOURCE_BUILT_SDK_ID_ARCADE=Microsoft.DotNet.Arcade.Sdk + export SOURCE_BUILT_SDK_VERSION_ARCADE=$ARCADE_BOOTSTRAP_VERSION + export SOURCE_BUILT_SDK_DIR_ARCADE=$restoredPackagesDir/ArcadeBootstrapPackage/microsoft.dotnet.arcade.sdk/$ARCADE_BOOTSTRAP_VERSION +fi + +sourceLinkLine=`grep -m 1 'MicrosoftSourceLinkCommonVersion' "$packageVersionsPath"` +versionPattern="(.*)" +if [[ $sourceLinkLine =~ $versionPattern ]]; then + export SOURCE_LINK_BOOTSTRAP_VERSION=${BASH_REMATCH[1]} +fi + +echo "Found bootstrap SDK $SDK_VERSION, bootstrap Arcade $ARCADE_BOOTSTRAP_VERSION, bootstrap SourceLink $SOURCE_LINK_BOOTSTRAP_VERSION" + +export DOTNET_CLI_TELEMETRY_OPTOUT=1 +export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 +export NUGET_PACKAGES=$restoredPackagesDir/ + +if [ "$alternateTarget" == "true" ]; then + "$CLI_ROOT/dotnet" $CLI_ROOT/sdk/$SDK_VERSION/MSBuild.dll "$SCRIPT_ROOT/build.proj" /bl:source-build-test.binlog /clp:v=m ${MSBUILD_ARGUMENTS[@]} "$@" +else + LogDateStamp=$(date +"%m%d%H%M%S") + $CLI_ROOT/dotnet $CLI_ROOT/sdk/$SDK_VERSION/MSBuild.dll /bl:$SCRIPT_ROOT/artifacts/log/Debug/BuildXPlatTasks_$LogDateStamp.binlog $SCRIPT_ROOT/tools-local/init-build.proj /t:PrepareOfflineLocalTools ${MSBUILD_ARGUMENTS[@]} "$@" + + $CLI_ROOT/dotnet $CLI_ROOT/sdk/$SDK_VERSION/MSBuild.dll /bl:$SCRIPT_ROOT/artifacts/log/Debug/Build_$LogDateStamp.binlog $SCRIPT_ROOT/build.proj ${MSBUILD_ARGUMENTS[@]} "$@" +fi diff --git a/src/SourceBuild/tarball/content/eng/Build.props b/src/SourceBuild/tarball/content/eng/Build.props new file mode 100644 index 000000000..d8414bdbf --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/Build.props @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/eng/No.proj b/src/SourceBuild/tarball/content/eng/No.proj new file mode 100644 index 000000000..9ba857dd2 --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/No.proj @@ -0,0 +1,9 @@ + + + + + + diff --git a/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.AllowedPrebuilts.targets b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.AllowedPrebuilts.targets new file mode 100644 index 000000000..f4191b490 --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.AllowedPrebuilts.targets @@ -0,0 +1,11 @@ + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.DestructiveCleanup.targets b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.DestructiveCleanup.targets new file mode 100644 index 000000000..18ef94a13 --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.DestructiveCleanup.targets @@ -0,0 +1,63 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.KnownExtraPrebuilts.targets b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.KnownExtraPrebuilts.targets new file mode 100644 index 000000000..fc10206fc --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.KnownExtraPrebuilts.targets @@ -0,0 +1,136 @@ + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.TextOnlyPrebuilts.targets b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.TextOnlyPrebuilts.targets new file mode 100644 index 000000000..efc0276b4 --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.TextOnlyPrebuilts.targets @@ -0,0 +1,99 @@ + + + + + + + + $([MSBuild]::NormalizeDirectory('$(TarballRootDir)', 'packages', 'text-only')) + + + + + + + + + + + + + + + + + + + + $([System.IO.Path]::GetDirectoryName('$(TextOnlyPackageRootDir)')) + $([System.IO.Path]::GetDirectoryName('$(PackageVersionDirFile)')) + $([System.IO.Path]::GetFileName('$(PackageVersionDirFile)')) + $([System.IO.Path]::GetFileName('$(PackageIdDirFile)')) + + $([MSBuild]::NormalizeDirectory('$(TextOnlyPackageBaseDir)', '$(PackageId)', '$(PackageVersion)')) + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.targets b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.targets new file mode 100644 index 000000000..3948a9cc6 --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.targets @@ -0,0 +1,386 @@ + + + + + + + + + + + + + $([MSBuild]::NormalizeDirectory('$(TarballRoot)')) + $([MSBuild]::NormalizeDirectory('$(TarballRootDir)', 'src')) + $([MSBuild]::NormalizeDirectory('$(TarballRootDir)', 'packages', 'prebuilt')) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(TarballRootDir)source-build-info.txt + + + + + + + + $(ProjectDir)tools-local\tasks\Microsoft.DotNet.SourceBuild.Tasks.LeakDetection\Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj + + + + + + + + + + + + + + + + + + + + + + + + Detected forbidden prebuilts. They must be removed, or explicitly allowed (see target for details): + $(PrebuiltErrorText)%0A@(ForbiddenPrebuiltPackageFile -> '%(Identity)', '%0A') + + + + + + diff --git a/src/SourceBuild/tarball/content/eng/Versions.props b/src/SourceBuild/tarball/content/eng/Versions.props new file mode 100644 index 000000000..d8bab4660 --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/Versions.props @@ -0,0 +1,26 @@ + + + + + 6 + 0 + 0-preview.6 + 100-preview.6 + $(MajorVersion).$(MinorVersion).$(RuntimePatchVersion) + $(MajorVersion).$(MinorVersion).$(RuntimePatchVersion) + $(MajorVersion).$(MinorVersion).$(SdkPatchVersion) + + + + 0.1.0 + alpha.1 + + + + 2.2.0 + + + + 0.1.0-6.0.100-bootstrap.3 + + diff --git a/src/SourceBuild/tarball/content/eng/install-nuget-credprovider.sh b/src/SourceBuild/tarball/content/eng/install-nuget-credprovider.sh new file mode 100644 index 000000000..880105e3c --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/install-nuget-credprovider.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# This script installs the NuGet Credential Provider. It is intended for use on CI machines only. + +# Originally copied from https://github.com/dotnet/core-setup/blob/aa28510afc9b986c6837db6784d816fe4a66c7d0/eng/install-nuget-credprovider.sh + +set -e + +# Install curl if necessary. Dependency exists inside downloaded script. +if command -v curl > /dev/null; then + echo "curl found." +else + echo "curl not found, trying to install..." + ( + set +e + set -x + + apt update && apt install -y curl + + apk update && apk upgrade && apk add curl + + exit 0 + ) +fi + +# Install. Ported from https://gist.github.com/shubham90/ad85f2546a72caa20d57bce03ec3890f +install_credprovider() { + # Download the provider and install. + cred_provider_url='https://raw.githubusercontent.com/Microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh' + curl "$cred_provider_url" -s -S -L | bash + + # Environment variable to enable session token cache. More on this here: https://github.com/Microsoft/artifacts-credprovider#help + export NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED=true +} + +install_credprovider + +# Additional setup to try to avoid flakiness: https://github.com/dotnet/arcade/issues/3932 +export DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER=0 +export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20 +export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20 diff --git a/src/SourceBuild/tarball/content/global.json b/src/SourceBuild/tarball/content/global.json new file mode 100644 index 000000000..f48709aa6 --- /dev/null +++ b/src/SourceBuild/tarball/content/global.json @@ -0,0 +1,12 @@ +{ + "tools": { + "dotnet": "6.0.100-preview.5.21225.11" + }, + "msbuild-sdks": { + "Microsoft.Build.CentralPackageVersions": "2.0.1", + "Microsoft.Build.Traversal": "2.0.2", + "Microsoft.NET.Sdk.IL": "3.0.0-preview-27107-01", + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21304.1", + "Yarn.MSBuild": "1.15.2" + } +} diff --git a/src/SourceBuild/tarball/content/keys/Newtonsoft.Json.snk b/src/SourceBuild/tarball/content/keys/Newtonsoft.Json.snk new file mode 100644 index 000000000..c26e9e4ea Binary files /dev/null and b/src/SourceBuild/tarball/content/keys/Newtonsoft.Json.snk differ diff --git a/src/SourceBuild/tarball/content/keys/NuGet.Client.snk b/src/SourceBuild/tarball/content/keys/NuGet.Client.snk new file mode 100644 index 000000000..695f1b387 Binary files /dev/null and b/src/SourceBuild/tarball/content/keys/NuGet.Client.snk differ diff --git a/src/SourceBuild/tarball/content/keys/README.md b/src/SourceBuild/tarball/content/keys/README.md new file mode 100644 index 000000000..d83cf9491 --- /dev/null +++ b/src/SourceBuild/tarball/content/keys/README.md @@ -0,0 +1,13 @@ +This directory contains the public key portion of keys used by different +projects so we can public sign (also known as OSS signing) assemblies that need +to be signed to have the correct strong name. + +These are used for projects which full sign their assemblies and don't have keys +checked in. + +To extract a key, take an existing built binary for a project (e.g. download a +nupkg from NuGet.org and then unpack one of the assemblies from it) and use `sn`: + +``` +sn -e +``` diff --git a/src/SourceBuild/tarball/content/prep.sh b/src/SourceBuild/tarball/content/prep.sh new file mode 100755 index 000000000..58cf90e40 --- /dev/null +++ b/src/SourceBuild/tarball/content/prep.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)" + +usage() { + echo "usage: $0" + echo "" + echo " Prepares a tarball to be built by downloading Private.SourceBuilt.Artifacts.*.tar.gz and" + echo " installing the version of dotnet referenced in global.json" + echo "" +} + +positional_args=() +while :; do + if [ $# -le 0 ]; then + break + fi + lowerI="$(echo "$1" | awk '{print tolower($0)}')" + case $lowerI in + "-?"|-h|--help) + usage + exit 0 + ;; + *) + positional_args+=("$1") + ;; + esac + + shift +done + +# Check for the archive text file which describes the location of the archive files to download +if [ ! -f $SCRIPT_ROOT/packages/archive/archiveArtifacts.txt ]; then + echo " ERROR: $SCRIPT_ROOT/packages/archive/archiveArtifacts.txt does not exist. Cannot determine which archives to download. Exiting..." + exit -1 +fi + +downloadArtifacts=true +installDotnet=true + +# Check to make sure curl exists to download the archive files +if ! command -v curl &> /dev/null +then + echo " ERROR: curl not found. Exiting..." + exit -1 +fi + +# Check if Private.SourceBuilt artifacts archive exists +if [ -f $SCRIPT_ROOT/packages/archive/Private.SourceBuilt.Artifacts.*.tar.gz ]; then + echo " Private.SourceBuilt.Artifacts.*.tar.gz exists...it will not be downloaded" + downloadArtifacts=false +fi + +# Check if dotnet is installed +if [ -d $SCRIPT_ROOT/.dotnet ]; then + echo " ./.dotnet SDK directory exists...it will not be installed" + installDotnet=false; +fi + +# Read the archive text file to get the archives to download and download them +while read -r line; do + if [[ $line == *"Private.SourceBuilt.Artifacts"* ]]; then + if [ "$downloadArtifacts" == "true" ]; then + echo " Downloading source-built artifacts..." + (cd $SCRIPT_ROOT/packages/archive/ && curl -O $line) + fi + fi +done < $SCRIPT_ROOT/packages/archive/archiveArtifacts.txt + +# Check for the version of dotnet to install +if [ "$installDotnet" == "true" ]; then + echo " Installing dotnet..." + (source ./eng/common/tools.sh && InitializeDotNetCli true) +fi diff --git a/src/SourceBuild/tarball/content/repos/Directory.Build.props b/src/SourceBuild/tarball/content/repos/Directory.Build.props new file mode 100644 index 000000000..851855392 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/Directory.Build.props @@ -0,0 +1,148 @@ + + + + $(MSBuildProjectName) + + + + + + + + + $(RepositoryName) + $(ClonedSubmoduleDirectory)$(SourceDirectory).$(GitCommitHash)/ + $(SubmoduleDirectory)$(SourceDirectory).$(GitCommitHash)/ + true + $(LoggingDir)$(RepositoryName).log + >> $(RepoConsoleLogFile) 2>&1 + true + + + $(CompletedSemaphorePath)$(RepositoryName)/ + + + + - + 0 + + + -- + false + + + + $(GitCommitDate.Replace('-', '')) + + + + + '$(RepositoryName)' + $(ProjectBuildReason) to produce tarball + $(ProjectBuildReason) in tarball + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(PackagesDir) + $(ArcadeBootstrapPackageDir) + $(ARCADE_BOOTSTRAP_VERSION) + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/Directory.Build.targets b/src/SourceBuild/tarball/content/repos/Directory.Build.targets new file mode 100644 index 000000000..91de0162c --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/Directory.Build.targets @@ -0,0 +1,756 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + true + $(ProjectDirectory)/eng/common/tools.sh + $(ProjectDirectory)/eng/common/build.sh + + + + $(RepoApiImplemented) + $(RepoApiImplemented) + $(RepoApiImplemented) + + + + $(ProjectDirectory)NuGet.config + $(ProjectDirectory)NuGet.Config + $(ProjectDirectory)src\NuGet.config + $(ProjectDirectory)src\NuGet.Config + + + + + $(RepoApiArgs) /p:DotNetPackageVersionPropsPath=$(PackageVersionPropsPath) + + + + $(RepoApiArgs) /p:DotNetRestoreSourcePropsPath=$(RestoreSourcePropsPath) + $(RepoApiArgs) /p:DotNetBuildOffline=true + + + + $(RepoApiArgs) /p:DotNetOutputBlobFeedDir=$(SourceBuiltBlobFeedDir) + + + + + + <_DependentProject Include="@(RepositoryReference -> '%(Identity).proj')" /> + + + + + + + + + + + + + git --work-tree=$(ProjectDirectory) apply --ignore-whitespace --whitespace=nowarn + + + + + + + + + + + + + + + + + + logger_path="%24toolset_dir"/%24%28cd "$toolset_dir" && find . -name Microsoft.DotNet.Arcade.Sdk.dll \( -regex '.*netcoreapp2.1.*' -or -regex '.*net5.0.*' \) ) + + + + logger_path="%24toolset_dir"/%24%28cd "$toolset_dir" && find . -name Microsoft.DotNet.ArcadeLogging.dll \( -regex '.*netcoreapp2.1.*' -or -regex '.*net5.0.*' \) ) + + + + $(ArcadeLoggingReplacementText) + if [[ ! -f $logger_path ]]; then + $(ArcadeSdkReplacementText) + fi + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_PossibleCliVersionJsonPath Include="sdk.version" /> + <_PossibleCliVersionJsonPath Include="tools.dotnet" /> + + + + + + + + + + <_PackageVersionPropsBackupPath>$(PackageVersionPropsPath).pre.$(RepositoryName).xml + + + + + <_AdditionalAssetDirs Include="$(SourceBuiltToolsetDir)" Condition="Exists('$(SourceBuiltToolsetDir)')" /> + + + + + + + + + + + + + + <_KnownOriginPackagePaths Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" /> + <_KnownOriginPackagePaths Include="$(PrebuiltPackagesPath)*.nupkg" /> + <_KnownOriginPackagePaths Include="$(ReferencePackagesDir)*.nupkg" /> + + + + + + + + + + + + <_ReportDir>$(ConflictingPackageReportDir)before-$(RepositoryName)/ + <_ReportDataFile>$(_ReportDir)usage.xml + <_ProjectAssetsJsonArchiveFile>$(_ReportDir)all-project-assets-json-files.zip + + + + + + + + + + + + + + + <_DotNetRestoreSources Include="$(ExtraRestoreSourcePath)" Condition="'$(ExtraRestoreSourcePath)' != ''"/> + <_DotNetRestoreSources Include="$(SourceBuiltPackagesPath)" /> + <_DotNetRestoreSources Include="$(ReferencePackagesDir)" Condition="'$(OfflineBuild)' == 'true'"/> + <_DotNetRestoreSources Include="$(PrebuiltPackagesPath)" Condition="'$(OfflineBuild)' == 'true'"/> + <_DotNetRestoreSources Include="$(PrebuiltSourceBuiltPackagesPath)" Condition="'$(OfflineBuild)' == 'true'"/> + + <_AdditionalAssetDirs Include="$(SourceBuiltToolsetDir)" Condition="Exists('$(SourceBuiltToolsetDir)')" /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(BuildCommand) /v:$(LogVerbosity) $(RepoApiArgs) $(RedirectRepoOutputToLog) + $(BuildCommand) $(RepoApiArgs) $(RedirectRepoOutputToLog) + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_BuiltPackages Condition="'$(PackagesOutput)' != ''" Include="$(PackagesOutput)/*.nupkg" Exclude="$(PackagesOutput)/*.symbols.nupkg"/> + <_BuiltPackages Condition="'@(PackagesOutputList)' != ''" Include="%(PackagesOutputList.Identity)/*.nupkg" Exclude="%(PackagesOutputList.Identity)/*.symbols.nupkg"/> + + + + + + <_BuiltIntermediatePackages Condition="'$(PackagesOutput)' != ''" Include="$(PackagesOutput)/Microsoft.SourceBuild.Intermediate.*.nupkg" Exclude="$(PackagesOutput)/*.symbols.nupkg"/> + <_BuiltIntermediatePackages Condition="'@(PackagesOutputList)' != ''" Include="%(PackagesOutputList.Identity)/Microsoft.SourceBuild.Intermediate.*.nupkg" Exclude="%(PackagesOutputList.Identity)/*.symbols.nupkg"/> + + + + + <_DestinationPath>$(SourceBuiltPackagesPath) + + <_DestinationPath Condition="$([System.String]::Copy(%(_BuiltIntermediatePackages.Identity)).Contains('source-build-reference-packages'))">$(ReferencePackagesDir) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_FilesToCopy Include="$(PackagesDir)$([System.String]::copy('%(_BuiltPackageInfos.PackageId)').ToLower())/%(_BuiltPackageInfos.PackageVersion)/**/*.nupkg" /> + <_FilesToDelete Include="$(PackagesDir)$([System.String]::copy('%(_BuiltPackageInfos.PackageId)').ToLower())/%(_BuiltPackageInfos.PackageVersion)/**/*.*" /> + + + + + + + + + + <_ToolPackage + Condition="'%(BuiltSdkPackageOverride.Version)' == ''" + Include="$(SourceBuiltPackagesPath)%(BuiltSdkPackageOverride.Identity)*.nupkg" + Exclude="$(SourceBuiltPackagesPath)%(BuiltSdkPackageOverride.Identity)*.symbols.nupkg" + Id="%(BuiltSdkPackageOverride.Identity)" /> + <_ToolPackage + Condition="'%(BuiltSdkPackageOverride.Version)' != ''" + Include="$(SourceBuiltPackagesPath)%(BuiltSdkPackageOverride.Identity).%(BuiltSdkPackageOverride.Version).nupkg" + Exclude="$(SourceBuiltPackagesPath)%(BuiltSdkPackageOverride.Identity).%(BuiltSdkPackageOverride.Version).symbols.nupkg" + Id="%(BuiltSdkPackageOverride.Identity)" /> + + + + + + + + + + + + + + + + + + + + + + + + <_PackagesNotCreatedReason Include="^ There may have been a silent failure in the submodule build. To confirm, check the build log file for undetected errors that may have prevented package creation: $(RepoConsoleLogFile)" /> + <_PackagesNotCreatedReason Include="^ This error might be a false positive if $(RepositoryName) intentionally builds no nuget packages. If so, set the SkipEnsurePackagesCreated property to true in $(MSBuildProjectFullPath)" /> + <_PackagesNotCreatedReason Include="^ The 'bin' directory might be dirty from a previous build and the package files already existed. If so, perform a clean build, or check which packages were already in 'bin' by opening $(_PackageVersionPropsBackupPath)" /> + <_PackagesNotCreatedReason Include="^ The packages may have been written to an unexpected directory. For example, some repos used bin/ and changed to artifacts/ to match Arcade. Check PackagesOutput in $(MSBuildProjectFullPath) (currently '$(PackagesOutput)')" /> + + + + + + + + + + + + + + + + + + + + + $(LocalNuGetPackagesRoot)$(RepositoryName)/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + false + + + + + + + + + + false + true + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/application-insights.proj b/src/SourceBuild/tarball/content/repos/application-insights.proj new file mode 100644 index 000000000..f62473e51 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/application-insights.proj @@ -0,0 +1,44 @@ + + + + ApplicationInsights-dotnet + + + + + + $(ProjectDirectory)/bin/$(Configuration) + false + true + + + + + + + $(ProjectDirectory)/Microsoft.ApplicationInsights.csproj + $(BuildCommandArgs) /p:Configuration=$(Configuration) + + $(BuildCommandArgs) /p:EnlistmentRoot=$(ProjectDirectory)/src + $(BuildCommandArgs) /p:RelativeOutputPathBase= + $(BuildCommandArgs) /v:$(LogVerbosity) + $(BuildCommandArgs) $(RedirectRepoOutputToLog) + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/arcade.proj b/src/SourceBuild/tarball/content/repos/arcade.proj new file mode 100644 index 000000000..719b3e50d --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/arcade.proj @@ -0,0 +1,78 @@ + + + + + + + $(BuildCommandArgs) $(FlagParameterPrefix)pack + $(BuildCommandArgs) $(FlagParameterPrefix)configuration $(Configuration) + $(BuildCommandArgs) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg) + + true + $(BuildCommandArgs) -v $(LogVerbosity) + $(BuildCommandArgs) -bl + $(BuildCommandArgs) -ci + $(BuildCommandArgs) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg) + $(BuildCommandArgs) /p:ArcadeBuildFromSource=true + $(BuildCommandArgs) /p:CopyWipIntoInnerSourceBuildRepo=true + + $(ProjectDirectory)\build$(ShellExtension) $(BuildCommandArgs) + + + $(ProjectDirectory)/NuGet.config + + $(ProjectDirectory)global.json + + true + false + + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + + + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + %(PreviouslySourceBuiltNuGetPackageInfo.PackageVersion) + $(BuildCommand) /p:NuGetVersion=$(PreviouslySourceBuiltNuGetVersion) + + + + + diff --git a/src/SourceBuild/tarball/content/repos/aspnet-xdt.proj b/src/SourceBuild/tarball/content/repos/aspnet-xdt.proj new file mode 100644 index 000000000..c51910bad --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/aspnet-xdt.proj @@ -0,0 +1,33 @@ + + + + xdt + + + + + + + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) --binaryLog + $(BuildCommandArgs) -ci + $(BuildCommandArgs) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg) + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)global.json + $(ProjectDirectory)/NuGet.config + true + false + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/aspnetcore.proj b/src/SourceBuild/tarball/content/repos/aspnetcore.proj new file mode 100644 index 000000000..50e459d14 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/aspnetcore.proj @@ -0,0 +1,93 @@ + + + aspnetcore + + + + + + $(TargetRid) + freebsd-x64 + osx-x64 + + $(BuildCommandArgs) --restore --build --pack + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) --ci + $(BuildCommandArgs) -bl + $(BuildCommandArgs) /v:$(LogVerbosity) + + $(BuildCommandArgs) --arch $(Platform) + $(BuildCommandArgs) /p:BuildNodeJs=false + $(BuildCommandArgs) /p:SourceBuildRuntimeIdentifier=$(OverrideTargetRid) + $(BuildCommandArgs) /p:UseAppHost=false + $(BuildCommandArgs) /p:PublishCompressedFilesPathPrefix=$(SourceBuiltAspNetCoreRuntime) + + + $(BuildCommandArgs) /p:MicrosoftNetFrameworkReferenceAssembliesVersion=1.0.0 + + true + + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + false + true + + $(ProjectDirectory)global.json + $(ProjectDirectory)NuGet.config + + $(SourceBuiltPackagesPath) + $(EnvironmentRestoreSources)%3B$(ReferencePackagesDir)%3B$(PrebuiltPackagesPath) + $(EnvironmentRestoreSources)%3Bhttps://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public%40Local/nuget/v3/index.json + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/clicommandlineparser.proj b/src/SourceBuild/tarball/content/repos/clicommandlineparser.proj new file mode 100644 index 000000000..19a01f0a9 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/clicommandlineparser.proj @@ -0,0 +1,41 @@ + + + + CliCommandLineParser + + + + + + + $(FlagParameterPrefix)pack $(FlagParameterPrefix)configuration $(Configuration) + $(BuildCommandArgs) --projects $(ProjectDirectory)CommandLine.sln + $(BuildCommandArgs) $(FlagParameterPrefix)restore + + + true + $(BuildCommandArgs) -v $(LogVerbosity) + $(BuildCommandArgs) -bl + $(BuildCommandArgs) -ci + + $(BuildCommandArgs) /p:DotNetPackageVersionPropsPath=$(PackageVersionPropsPath) + + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + false + $(ProjectDirectory)global.json + + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + false + $(ProjectDirectory)/NuGet.config + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/command-line-api.proj b/src/SourceBuild/tarball/content/repos/command-line-api.proj new file mode 100644 index 000000000..57e458792 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/command-line-api.proj @@ -0,0 +1,30 @@ + + + + + + --restore --build --pack + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) --binaryLog + $(BuildCommandArgs) -ci + $(BuildCommandArgs) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg) + $(BuildCommandArgs) /p:Projects=$(ProjectDirectory)source-build.slnf + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)global.json + $(ProjectDirectory)NuGet.config + true + false + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/common.proj b/src/SourceBuild/tarball/content/repos/common.proj new file mode 100644 index 000000000..2c04a0bfa --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/common.proj @@ -0,0 +1,24 @@ + + + + + aspnet + servicing/1.0.x + $(ProjectDirectory)artifacts/build/ + /p:Configuration=$(Configuration) + $(ProjectDirectory)/build$(ShellExtension) -NoTest $(BuildArguments) + $(ProjectDirectory)/clean$(ShellExtension) $(BuildArguments) + false + true + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/cssparser.proj b/src/SourceBuild/tarball/content/repos/cssparser.proj new file mode 100644 index 000000000..2bd2e2608 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/cssparser.proj @@ -0,0 +1,37 @@ + + + + + + $(ProjectDirectory)/src/Microsoft.Css.Parser/bin/$(Configuration)/ + false + true + + + + + + + $(ProjectDirectory)/src/Microsoft.Css.Parser/Microsoft.Css.Parser.csproj + $(BuildCommandArgs) /p:Configuration=$(Configuration) + $(BuildCommandArgs) /v:$(LogVerbosity) + $(BuildCommandArgs) $(RedirectRepoOutputToLog) + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/diagnostics.proj b/src/SourceBuild/tarball/content/repos/diagnostics.proj new file mode 100644 index 000000000..169fe0965 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/diagnostics.proj @@ -0,0 +1,43 @@ + + + + + + $(ProjectDirectory)global.json + $(ProjectDirectory)NuGet.config + $(ProjectDirectory)/artifacts/packages/$(Configuration)/Shipping + false + true + + + + + + + + + + + $(ProjectDirectory)/src/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj + $(BuildCommandArgs) /p:Configuration=$(Configuration) + $(BuildCommandArgs) /v:$(LogVerbosity) + $(BuildCommandArgs) $(RedirectRepoOutputToLog) + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/fsharp.proj b/src/SourceBuild/tarball/content/repos/fsharp.proj new file mode 100644 index 000000000..22dd1b7e7 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/fsharp.proj @@ -0,0 +1,39 @@ + + + + + --restore --build --pack --publish + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) --binaryLog + $(BuildCommandArgs) --ci + $(BuildCommandArgs) /p:FSharpSourceBuild=true + $(BuildCommandArgs) /p:DotNetBuildFromSource=true + $(BuildCommandArgs) /p:TreatWarningsAsErrors=false + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + false + $(ProjectDirectory)global.json + $(ProjectDirectory)/NuGet.config + true + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/humanizer.proj b/src/SourceBuild/tarball/content/repos/humanizer.proj new file mode 100644 index 000000000..a4cb2a6b2 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/humanizer.proj @@ -0,0 +1,48 @@ + + + + Humanizer + + + + + + $(ProjectDirectory)/src/Humanizer/bin/$(Configuration) + false + true + $(ProjectDirectory)/src/NuGet.config + + + + + + + + + + + $(ProjectDirectory)src/Humanizer/Humanizer.csproj + $(BuildCommandArgs) /p:Configuration=$(Configuration) + $(BuildCommandArgs) /v:$(LogVerbosity) + $(BuildCommandArgs) $(RedirectRepoOutputToLog) + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/installer.proj b/src/SourceBuild/tarball/content/repos/installer.proj new file mode 100644 index 000000000..fd8a95be0 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/installer.proj @@ -0,0 +1,135 @@ + + + installer + + + + + + $(TargetRid) + osx-x64 + $(OverrideTargetRid.Substring(0, $(OverrideTargetRid.IndexOf("-")))) + + --runtime-id $(OverrideTargetRid) + --runtime-id $(TargetRid) + + --restore --build --pack --ci --binaryLog + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) $(RuntimeArg) + + + $(BuildCommandArgs) /p:NETCoreAppMaximumVersion=99.9 + $(BuildCommandArgs) /p:OSName=$(OSNameOverride) + $(BuildCommandArgs) /p:Rid=$(TargetRid) + $(BuildCommandArgs) /p:DOTNET_INSTALL_DIR=$(DotNetCliToolDir) + + + $(BuildCommandArgs) /p:AspNetCoreSharedFxInstallerRid=linux-$(Platform) + + $(BuildCommandArgs) /p:CoreSetupRid=freebsd-x64 /p:PortableBuild=true + $(BuildCommandArgs) /p:CoreSetupRid=osx-x64 + $(BuildCommandArgs) /p:CoreSetupRid=$(TargetRid) + + + $(BuildCommandArgs) /p:CoreSetupBlobRootUrl=file:%2F%2F$(LocalBlobStorageRoot) + + + $(BuildCommandArgs) /p:DotnetToolsetBlobRootUrl=file:%2F%2F$(LocalBlobStorageRoot) + + $(BuildCommandArgs) /p:SkipBuildingInstallers=true + $(BuildCommandArgs) /p:IncludeNuGetPackageArchive=false + $(BuildCommandArgs) /p:IncludeAdditionalSharedFrameworks=false + $(BuildCommandArgs) /p:UsePortableLinuxSharedFramework=false + $(BuildCommandArgs) /p:IncludeSharedFrameworksForBackwardsCompatibilityTests=false + + $(ProjectDirectory)/build$(ShellExtension) $(BuildCommandArgs) + + win-x64 + osx-x64 + freebsd-x64 + $(TargetRid) + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + $(PackagesOutput) + false + true + true + + $(SourceBuiltPackagesPath) + $(EnvironmentExternalRestoreSources)%3B$(ReferencePackagesDir)%3B$(PrebuiltPackagesPath) + + + true + $(ProjectDirectory)/NuGet.config + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/known-good-tests.proj b/src/SourceBuild/tarball/content/repos/known-good-tests.proj new file mode 100644 index 000000000..e8d75dd4b --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/known-good-tests.proj @@ -0,0 +1,37 @@ + + + + + false + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/known-good.proj b/src/SourceBuild/tarball/content/repos/known-good.proj new file mode 100644 index 000000000..6d7f42094 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/known-good.proj @@ -0,0 +1,43 @@ + + + + + false + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/linker.proj b/src/SourceBuild/tarball/content/repos/linker.proj new file mode 100644 index 000000000..823dde083 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/linker.proj @@ -0,0 +1,81 @@ + + + + + + Microsoft.NET.ILLink.Tasks + + $(BuildCommandArgs) $(FlagParameterPrefix)ci + $(BuildCommandArgs) $(FlagParameterPrefix)configuration $(Configuration) + $(BuildCommandArgs) $(FlagParameterPrefix)restore + $(BuildCommandArgs) $(FlagParameterPrefix)build + $(BuildCommandArgs) $(FlagParameterPrefix)pack + $(BuildCommandArgs) $(FlagParameterPrefix)publish + $(BuildCommandArgs) -bl + + $(BuildCommandArgs) /p:ArcadeBuildFromSource=true + $(BuildCommandArgs) /p:CopyWipIntoInnerSourceBuildRepo=true + + $(ProjectDirectory)\build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + + false + $(ProjectDirectory)/NuGet.config + true + + + + + + + + + + + + + + $(ProjectDirectory)src/ILLink.Tasks/ILLink.Tasks.nuspec + + + + + $id$ + $version$ + $authors$ + $description$ + + + + + + + +]]> + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/msbuild.proj b/src/SourceBuild/tarball/content/repos/msbuild.proj new file mode 100644 index 000000000..d635daa70 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/msbuild.proj @@ -0,0 +1,105 @@ + + + + + $(ProjectDirectory)/artifacts/packages/$(Configuration)/Shipping + $(ProjectDirectory)/artifacts/packages/$(Configuration)/NonShipping + + $(OutputVersionArgs) /p:DisableNerdbankVersioning=true + + --build --publish --pack + $(BuildCommandArgs) /p:DotNetBuildFromSource=true + $(BuildCommandArgs) /p:DotNetCoreSdkDir=$(DotNetCliToolDir) + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) --projects $(ProjectDirectory)MSBuild.SourceBuild.slnf + $(BuildCommandArgs) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg) + $(BuildCommandArgs) $(OutputVersionArgs) + $(BuildCommandArgs) /p:DotNetPackageVersionPropsPath=$(PackageVersionPropsPath) + + + $(BuildCommandArgs) --warnAsError false + + true + $(BuildCommandArgs) -v $(LogVerbosity) + $(BuildCommandArgs) -bl + $(BuildCommandArgs) -ci + + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + + $(ProjectDirectory)/NuGet.config + + true + false + + $(ProjectDirectory)global.json + + $(SourceBuiltPackagesPath) + $(EnvironmentRestoreSources)%3B$(ReferencePackagesDir)%3B$(PrebuiltPackagesPath) + $(EnvironmentRestoreSources)%3Bhttps://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public%40Local/nuget/v3/index.json + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_CentralVersionsToolPackage + Include="$(ReferencePackagesDir)%(CentralPackageVersionsSdkOverride.Identity)*.nupkg" + Id="%(CentralPackageVersionsSdkOverride.Identity)" /> + + + + $(ToolPackageExtractDir)%(_CentralVersionsToolPackage.Id)/ + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/netcorecli-fsc.proj b/src/SourceBuild/tarball/content/repos/netcorecli-fsc.proj new file mode 100644 index 000000000..a9c1d27d3 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/netcorecli-fsc.proj @@ -0,0 +1,20 @@ + + + + + /bl /v:$(LogVerbosity) /p:OutputPath=$(OutputPath)$(RepositoryName)/ /p:BaseIntermediateOutputPath=$(IntermediatePath)$(RepositoryName) + $(DotnetToolCommand) pack -c $(Configuration) --output $(SourceBuiltPackagesPath) --no-build FSharp.NET.Sdk.csproj /p:NuspecFile=FSharp.NET.Sdk.nuspec $(OutputArgs) $(RedirectRepoOutputToLog) + $(SourceBuiltPackagesPath) + false + $(SubmoduleDirectory)$(RepositoryName)/ + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/newtonsoft-json.proj b/src/SourceBuild/tarball/content/repos/newtonsoft-json.proj new file mode 100644 index 000000000..1bebd52f0 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/newtonsoft-json.proj @@ -0,0 +1,39 @@ + + + + Newtonsoft.Json + + + + + + + + + + + + + + $(ProjectDirectory)/Src/NuGet.Config + $(KeysDir)Newtonsoft.Json.snk + $(ProjectDirectory)/Src/Newtonsoft.Json/ + $(NewtonsoftJsonDirectory)Newtonsoft.Json.csproj + "/p:DotnetOnly=true" "/p:Configuration=$(Configuration)" "/p:AssemblyOriginatorKeyFile=$(NewtonsoftJsonKeyFilePath)" "/p:SignAssembly=true" "/p:PublicSign=true" "/p:TreatWarningsAsErrors=false" "/p:AdditionalConstants=SIGNED" + $(DotnetToolCommand) build $(NewtonsoftJsonProjectPath) /bl:build.binlog $(DotnetToolCommandArguments) + $(DotnetToolCommand) pack $(NewtonsoftJsonProjectPath) /bl:pack.binlog $(DotnetToolCommandArguments) + $(DotnetToolCommand) clean $(NewtonsoftJsonProjectPath) $(DotnetToolCommandArguments) + $(NewtonsoftJsonDirectory)bin/$(Configuration)/ + false + true + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/newtonsoft-json901.proj b/src/SourceBuild/tarball/content/repos/newtonsoft-json901.proj new file mode 100644 index 000000000..c87ec499a --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/newtonsoft-json901.proj @@ -0,0 +1,31 @@ + + + + Newtonsoft.Json + + + + + + $(ProjectDirectory)/Src/NuGet.Config + $(KeysDir)Newtonsoft.Json.snk + $(ProjectDirectory)/Src/Newtonsoft.Json/ + $(NewtonsoftJsonDirectory)Newtonsoft.Json.Dotnet.csproj + "/p:DotnetOnly=true" "/p:Configuration=$(Configuration)" "/p:AssemblyOriginatorKeyFile=$(NewtonsoftJsonKeyFilePath)" "/p:SignAssembly=true" "/p:PublicSign=true" "/p:TreatWarningsAsErrors=false" "/p:AdditionalConstants=SIGNED" + $(DotnetToolCommand) build $(NewtonsoftJsonProjectPath) $(DotnetToolCommandArguments) + $(DotnetToolCommand) pack $(NewtonsoftJsonProjectPath) $(DotnetToolCommandArguments) + $(DotnetToolCommand) clean $(NewtonsoftJsonProjectPath) $(DotnetToolCommandArguments) + $(NewtonsoftJsonDirectory)bin/$(Configuration)/ + false + true + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/nuget-client.proj b/src/SourceBuild/tarball/content/repos/nuget-client.proj new file mode 100644 index 000000000..326263038 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/nuget-client.proj @@ -0,0 +1,68 @@ + + + + nuget.client + + + + + + $(ProjectDirectory)artifacts/nupkgs/ + $(ProjectDirectory)src/NuGet.Core/ + $(ProjectDirectory)cli/ + $(KeysDir)NuGet.Client.snk + false + true + true + + + + + + + + + + + + + + + + $(DotnetToolCommand) msbuild $(ProjectDirectory)/build/build.proj + $(BuildCommandBase) /p:VisualStudioVerion=15.0 + $(BuildCommandBase) /p:Configuration=$(Configuration) + $(BuildCommandBase) /p:BuildRTM=false + $(BuildCommandBase) /p:BuildNumber=$(OfficialBuildId) + $(BuildCommandBase) /v:$(LogVerbosity) + $(BuildCommandBase) /p:TreatWarningsAsErrors=false + $(BuildCommandBase) /p:DotNetPackageVersionPropsPath=$(PackageVersionPropsPath) + + + + + + + + $(BuildCommandBase) /t:PackXPlat + $(PackCommand) /p:PackageOutputPath=$(PackagesOutput) + $(PackCommand) /p:NoPackageAnalysis=true + $(PackCommand) /flp:v=detailed + $(PackCommand) /v:$(LogVerbosity) + $(PackCommand) /bl:pack.binlog + $(PackCommand) $(RedirectRepoOutputToLog) + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/package-source-build.proj b/src/SourceBuild/tarball/content/repos/package-source-build.proj new file mode 100644 index 000000000..d5f3a16b6 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/package-source-build.proj @@ -0,0 +1,38 @@ + + + + + false + $(SubmoduleDirectory)$(RepositoryName)/ + true + $(PackageVersionPropsPath) + $(GennedPackageVersionPropsPath) + + + + + + + + + + + + + + + + + + + + + $(OutputPath)$(SourceBuiltArtifactsTarballName).$(VersionPrefix)-$(VersionSuffix).tar.gz + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/roslyn-analyzers.proj b/src/SourceBuild/tarball/content/repos/roslyn-analyzers.proj new file mode 100644 index 000000000..3d517f6ef --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/roslyn-analyzers.proj @@ -0,0 +1,38 @@ + + + + + --restore --build --pack + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) -v $(LogVerbosity) + $(BuildCommandArgs) -bl + $(BuildCommandArgs) -warnaserror false + $(BuildCommandArgs) /p:TreatWarningsAsErrors=false + + + $(BuildCommandArgs) /p:OfficialBuild=true + $(BuildCommandArgs) /p:DotNetPackageVersionPropsPath=$(PackageVersionPropsPath) + $(BuildCommandArgs) /p:DotNetBuildFromSource=true + $(BuildCommandArgs) /p:GitHubRepositoryName=roslyn-analyzers + $(BuildCommandArgs) /p:RepositoryUrl=git://github.com/dotnet/roslyn-analyzers + + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + false + true + true + $(ProjectDirectory)global.json + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/roslyn.proj b/src/SourceBuild/tarball/content/repos/roslyn.proj new file mode 100644 index 000000000..4b992ed3c --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/roslyn.proj @@ -0,0 +1,78 @@ + + + + + + + --restore --build --pack + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) -v $(LogVerbosity) + $(BuildCommandArgs) -bl + $(BuildCommandArgs) /p:TreatWarningsAsErrors=false + + + $(BuildCommandArgs) /p:OfficialBuild=true + $(BuildCommandArgs) /p:ApplyPartialNgenOptimization=false + $(BuildCommandArgs) /p:EnablePartialNgenOptimization=false + $(BuildCommandArgs) /p:PublishWindowsPdb=false + $(BuildCommandArgs) /p:DotNetPackageVersionPropsPath=$(PackageVersionPropsPath) + $(BuildCommandArgs) /p:DotNetBuildFromSource=true + + + $(BuildCommandArgs) /p:UsingToolMicrosoftNetCompilers=false + + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + false + true + $(ProjectDirectory)global.json + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/runtime-portable.proj b/src/SourceBuild/tarball/content/repos/runtime-portable.proj new file mode 100644 index 000000000..0faeb9afc --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/runtime-portable.proj @@ -0,0 +1,56 @@ + + + + runtime + true + false + + + + + + + + + + + + + + + + + + $(ProjectDirectory)artifacts/portableLog + $(ProjectDirectory)artifacts/portableObj + + + + + + + + + + + + + + + + + + + + + + + + + + <_BuiltPackages Include="$(ShippingPackagesOutput)/*linux*nupkg" Exclude="$(ShippingPackagesOutput)/*.symbols.nupkg" /> + <_BuiltPackages Include="$(NonShippingPackagesOutput)/*linux*nupkg" Exclude="$(NonShippingPackagesOutput)/*.symbols.nupkg" /> + + + + diff --git a/src/SourceBuild/tarball/content/repos/runtime.common.props b/src/SourceBuild/tarball/content/repos/runtime.common.props new file mode 100644 index 000000000..16e04fcd5 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/runtime.common.props @@ -0,0 +1,143 @@ + + + runtime + + + + + + + + $(PortableBuild) + true + + $(TargetRid) + osx-x64 + freebsd-x64 + win-x64 + linux-$(Platform) + + false + false + $(UseSystemLibraries) + + + + + + $(ProjectDirectory)global.json + true + + + $(CoreClrBuildArguments) -cmakeargs -DCLR_CMAKE_USE_SYSTEM_LIBUNWIND=TRUE + $(CoreClrBuildArguments) skipnuget cross -skiprestore cmakeargs -DFEATURE_GDBJIT=TRUE + $(CoreClrBuildArguments) -clang6.0 /p:PortableBuild=true + + + $(CoreClrBuildArguments) /p:PortableBuild=false + + $(CoreClrBuildArguments) /p:PackagesDir=$(PackagesDir) + $(CoreClrBuildArguments) /p:ContinuousIntegrationBuild=true + $(CoreClrBuildArguments) /p:PackageRid=$(OverrideTargetRid) + + $(CoreClrBuildArguments) /p:NoPgoOptimize=true + $(CoreClrBuildArguments) /p:KeepNativeSymbols=true + $(CoreClrBuildArguments) /p:RuntimeOS=$(OverrideTargetRid.Substring(0, $(OverrideTargetRid.IndexOf("-")))) + $(CoreClrBuildArguments) /p:RuntimeOS=$(OverrideTargetRid) + + + + $(LibrariesBuildArguments) $(FlagParameterPrefix)restore $(FlagParameterPrefix)build $(FlagParameterPrefix)pack /p:SkipTests=true + $(LibrariesBuildArguments) $(FlagParameterPrefix)restore $(FlagParameterPrefix)build /p:IncludeTestUtils=true + $(LibrariesBuildArguments) $(FlagParameterPrefix)restore $(FlagParameterPrefix)buildtests $(FlagParameterPrefix)test /p:IncludeTestUtils=true + $(LibrariesBuildArguments) /p:TestRspFile=$(TestExclusionsDir)corefx/linux.docker.rsp + $(LibrariesBuildArguments) /p:ConfigurationGroup=$(Configuration) + $(LibrariesBuildArguments) /p:PackageRid=$(OverrideTargetRid) + + $(LibrariesBuildArguments) /p:RuntimeOS=$(OverrideTargetRid.Substring(0, $(OverrideTargetRid.IndexOf("-")))) + $(LibrariesBuildArguments) /p:RuntimeOS=$(OverrideTargetRid) + $(LibrariesBuildArguments) /p:PortableBuild=$(OverridePortableBuild) + $(LibrariesBuildArguments) /p:BuildAllPackages=true + $(LibrariesBuildArguments) /p:BuildAllOOBPackages=true + $(LibrariesBuildArguments) /p:KeepNativeSymbols=true + $(LibrariesBuildArguments) /p:BuiltSdkPackageOverride="" + $(LibrariesBuildArguments) /p:MicrosoftNETCoreDotNetHostPackageVersion=$(DOTNET_HOST_BOOTSTRAP_VERSION) + $(LibrariesBuildArguments) /p:MicrosoftNETCoreDotNetHostPolicyPackageVersion=$(DOTNET_HOST_BOOTSTRAP_VERSION) + + + + + $(FlagParameterPrefix)restore $(FlagParameterPrefix)build + $(InstallerBuildArguments) /p:PortableBuild=$(OverridePortableBuild) + $(InstallerBuildArguments) /p:KeepNativeSymbols=true + $(InstallerBuildArguments) -cmakeargs -DCLR_CMAKE_USE_SYSTEM_LIBUNWIND=TRUE + $(InstallerBuildArguments) /p:TargetArchitecture=$(Platform) /p:DisableCrossgen=true /p:CrossBuild=true + $(InstallerBuildArguments) /p:BuildDebPackage=false + $(InstallerBuildArguments) /p:BuildAllPackages=true + $(InstallerBuildArguments) /p:RestoreAllBuildRids=false + $(InstallerBuildArguments) /p:OutputRid=$(OverrideTargetRid) + $(InstallerBuildArguments) /p:DotNetOutputBlobFeedDir=$(SourceBuiltBlobFeedDir) + $(InstallerBuildArguments) /p:PublishCompressedFilesPathPrefix=$(SourceBuiltRuntimeDir) + $(InstallerBuildArguments) /p:BuiltSdkPackageOverride="" + $(InstallerBuildArguments) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg) + + $(FlagParameterPrefix)arch $(Platform) + $(CommonBuildArguments) $(FlagParameterPrefix)configuration $(Configuration) + $(CommonBuildArguments) /p:RuntimeConfiguration=$(Configuration) + $(CommonBuildArguments) /p:MicrosoftNetFrameworkReferenceAssembliesVersion=1.0.0 + $(CommonBuildArguments) $(FlagParameterPrefix)binaryLog + $(CommonBuildArguments) $(FlagParameterPrefix)ci + $(CommonBuildArguments) $(FlagParameterPrefix)runtimeConfiguration $(Configuration) + $(CommonBuildArguments) $(FlagParameterPrefix)verbosity $(LogVerbosity) + + $(CommonBuildArguments) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg) + $(CommonBuildArguments) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg) + + $(ProjectDirectory)build-source-build$(ShellExtension) $(CommonBuildArguments) $(FlagParameterPrefix)coreclr-args $(CoreClrBuildArguments) $(FlagParameterPrefix)libraries-args $(LibrariesBuildArguments) $(FlagParameterPrefix)installer-args $(InstallerBuildArguments) $(FlagParameterPrefix)additional-args + $(ArmEnvironmentVariables) $(BuildCommand) + $(ProjectDirectory)/clean$(ShellExtension) + + + + + false + $(ProjectDirectory)bin/Product/$(TargetOS).$(Platform).$(Configuration)/.nuget/pkg + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + $(ProjectDirectory)/NuGet.config + true + + + + + $(SourceBuiltPackagesPath) + $(EnvironmentRestoreSources)%3B$(ShippingPackagesOutput) + $(EnvironmentRestoreSources)%3B$(NonShippingPackagesOutput) + $(EnvironmentRestoreSources)%3B$(ReferencePackagesDir)%3B$(PrebuiltPackagesPath) + $(EnvironmentRestoreSources)%3Bhttps://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public%40Local/nuget/v3/index.json + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/runtime.common.targets b/src/SourceBuild/tarball/content/repos/runtime.common.targets new file mode 100644 index 000000000..a6d5f88ce --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/runtime.common.targets @@ -0,0 +1,72 @@ + + + + + + + + + + + + + + + $(ProjectDirectory)pkg/Microsoft.NETCore.Platforms/runtime.json + + + + + + + + + $(ToolPackageExtractDir)coreclr-tools + so + so + dylib + please define AssemblyExtension for $(TargetOS) + + + + + + + + + + + + + + + + + + + <_builtRuntimePackages Include="$(SourceBuiltAssetsDir)*.symbols.nupkg" /> + <_builtRuntimePackages> + $([System.String]::Copy('%(FileName)').Replace('symbols', 'nupkg')) + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/runtime.proj b/src/SourceBuild/tarball/content/repos/runtime.proj new file mode 100644 index 000000000..f20c04e17 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/runtime.proj @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/sdk.proj b/src/SourceBuild/tarball/content/repos/sdk.proj new file mode 100644 index 000000000..e07149dab --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/sdk.proj @@ -0,0 +1,66 @@ + + + sdk + + + + + + --pack --configuration $(Configuration) + $(BuildCommandArgs) /p:PackageProjectUrl=https://github.com/dotnet/sdk + $(BuildCommandArgs) /p:PublishCompressedFilesPathPrefix=$(SourceBuiltToolsetDir) + + true + $(BuildCommandArgs) -v $(LogVerbosity) + $(BuildCommandArgs) -bl + $(BuildCommandArgs) -ci + $(BuildCommandArgs) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg) + $(BuildCommandArgs) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg) + $(BuildCommandArgs) /p:Projects=$(ProjectDirectory)source-build.slnf + + + $(BuildCommandArgs) /p:ProduceReferenceAssembly=false + + + $(BuildCommandArgs) /p:PB_PackageVersionPropsUrl=file:%2F%2F$(PackageVersionPropsPath) + + $(ProjectDirectory)\build$(ShellExtension) $(BuildCommandArgs) + + + $(ProjectDirectory)/NuGet.config + $(ProjectDirectory)global.json + + true + false + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + + $(SourceBuiltPackagesPath) + $(EnvironmentRestoreSources)%3B$(ReferencePackagesDir)%3B$(PrebuiltPackagesPath) + $(EnvironmentRestoreSources)%3Bhttps://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public%40Local/nuget/v3/index.json + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/source-build-reference-packages.proj b/src/SourceBuild/tarball/content/repos/source-build-reference-packages.proj new file mode 100644 index 000000000..0bfe4116c --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/source-build-reference-packages.proj @@ -0,0 +1,30 @@ + + + + + $(BuildCommandArgs) $(FlagParameterPrefix)ci + $(BuildCommandArgs) $(FlagParameterPrefix)configuration $(Configuration) + $(BuildCommandArgs) $(FlagParameterPrefix)restore + $(BuildCommandArgs) $(FlagParameterPrefix)build + $(BuildCommandArgs) $(FlagParameterPrefix)pack + $(BuildCommandArgs) $(FlagParameterPrefix)publish + $(BuildCommandArgs) -bl + + $(BuildCommandArgs) /p:ArcadeBuildFromSource=true + $(BuildCommandArgs) /p:CopyWipIntoInnerSourceBuildRepo=true + + $(ProjectDirectory)\build$(ShellExtension) $(BuildCommandArgs) + + false + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + + + true + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/sourcelink.proj b/src/SourceBuild/tarball/content/repos/sourcelink.proj new file mode 100644 index 000000000..e0b4cca88 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/sourcelink.proj @@ -0,0 +1,34 @@ + + + + + + $(BuildCommandArgs) $(FlagParameterPrefix)pack + $(BuildCommandArgs) $(FlagParameterPrefix)configuration $(Configuration) + $(BuildCommandArgs) $(FlagParameterPrefix)binaryLog + $(BuildCommandArgs) $(FlagParameterPrefix)ci + $(BuildCommandArgs) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg) + $(BuildCommandArgs) /p:ArcadeBuildFromSource=true + $(BuildCommandArgs) /p:CopyWipIntoInnerSourceBuildRepo=true + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)global.json + $(ProjectDirectory)NuGet.config + true + false + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + + + true + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/symreader.proj b/src/SourceBuild/tarball/content/repos/symreader.proj new file mode 100644 index 000000000..0e8284c7e --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/symreader.proj @@ -0,0 +1,47 @@ + + + + + + $(ProjectDirectory)global.json + $(ProjectDirectory)nuget.config + $(ProjectDirectory)/artifacts/packages/$(Configuration)/Shipping + false + true + + + + + + + + + + + + + + + $(ProjectDirectory)/src//Microsoft.DiaSymReader//Microsoft.DiaSymReader.csproj + $(BuildCommandArgs) /p:Configuration=$(Configuration) + $(BuildCommandArgs) /v:$(LogVerbosity) + $(BuildCommandArgs) $(RedirectRepoOutputToLog) + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/templating.proj b/src/SourceBuild/tarball/content/repos/templating.proj new file mode 100644 index 000000000..5cea61baf --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/templating.proj @@ -0,0 +1,53 @@ + + + + + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + false + + --restore --pack --configuration $(Configuration) $(OutputVersionArgs) + $(BuildCommandArgs) $(FlagParameterPrefix)warnasError $(ArcadeFalseBoolBuildArg) + $(BuildCommandArgs) /p:PackSpecific=true + $(BuildCommandArgs) /p:UseAppHost=false + + true + $(BuildCommandArgs) -v $(LogVerbosity) + $(BuildCommandArgs) -bl + $(BuildCommandArgs) --ci + $(BuildCommandArgs) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg) + + + $(BuildCommandArgs) /p:PB_PackageVersionPropsUrl=file:%2F%2F$(PackageVersionPropsPath) + + $(ProjectDirectory)\build$(ShellExtension) $(BuildCommandArgs) + + true + + $(SourceBuiltPackagesPath) + $(EnvironmentRestoreSources)%3B$(ReferencePackagesDir)%3B$(PrebuiltPackagesPath) + $(EnvironmentRestoreSources)%3Bhttps://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public%40Local/nuget/v3/index.json + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/test-templates.proj b/src/SourceBuild/tarball/content/repos/test-templates.proj new file mode 100644 index 000000000..2951834f1 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/test-templates.proj @@ -0,0 +1,29 @@ + + + + + + --restore --build --pack + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) --binaryLog + $(BuildCommandArgs) -ci + $(BuildCommandArgs) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg) + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)global.json + $(ProjectDirectory)NuGet.config + true + false + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/vstest.proj b/src/SourceBuild/tarball/content/repos/vstest.proj new file mode 100644 index 000000000..e1feeb741 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/vstest.proj @@ -0,0 +1,31 @@ + + + + + $(TargetRid) + freebsd-x64 + + -DotNetBuildFromSource + $(BuildCommandArgs) -DotNetCoreSdkDir $(DotNetCliToolDir) + $(BuildCommandArgs) -c $(Configuration) + $(BuildCommandArgs) -r $(OverrideTargetRid) + $(BuildCommandArgs) -v $(OutputPackageVersion) + $(BuildCommandArgs) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg) + + $(ProjectDirectory)/build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)/artifacts/$(Configuration)/packages + false + true + $(ProjectDirectory)global.json + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/xliff-tasks.proj b/src/SourceBuild/tarball/content/repos/xliff-tasks.proj new file mode 100644 index 000000000..849340be0 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/xliff-tasks.proj @@ -0,0 +1,35 @@ + + + + + pack + $(BuildCommandArgs) $(ProjectDirectory)src\XliffTasks\XliffTasks.csproj + $(BuildCommandArgs) /v:$(LogVerbosity) + $(BuildCommandArgs) /flp:Verbosity=Diag + $(BuildCommandArgs) /bl + $(BuildCommandArgs) /p:TreatWarningsAsErrors=false + $(BuildCommandArgs) $(RedirectRepoOutputToLog) + + $(DotnetToolCommand) $(BuildCommandArgs) + + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + false + true + true + true + + $(ProjectDirectory)/NuGet.config + $(SourceBuiltPackagesPath) + $(EnvironmentRestoreSources)%3B$(ReferencePackagesDir)%3B$(PrebuiltPackagesPath) + $(EnvironmentRestoreSources)%3Bhttps://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public%40Local/nuget/v3/index.json + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/scripts/bootstrap/buildbootstrapcli.sh b/src/SourceBuild/tarball/content/scripts/bootstrap/buildbootstrapcli.sh new file mode 100755 index 000000000..b27b5476e --- /dev/null +++ b/src/SourceBuild/tarball/content/scripts/bootstrap/buildbootstrapcli.sh @@ -0,0 +1,411 @@ +#!/usr/bin/env bash +set -e +set -u +set -o pipefail + +usage() +{ + echo "Builds a bootstrap CLI from sources" + echo "Usage: $0 [BuildType] -rid -seedcli [-os ] [-clang ] [-corelib ]" + echo "" + echo "Options:" + echo " BuildType Type of build (-debug, -release), default: -release" + echo " -clang Override of the version of clang compiler to use" + echo " -corelib Path to System.Private.CoreLib.dll, default: use the System.Private.CoreLib.dll from the seed CLI" + echo " -os Operating system (used for corefx build), default: Linux" + echo " -rid Runtime identifier including the architecture part (e.g. rhel.6-x64)" + echo " -seedcli Seed CLI used to generate the target CLI" + echo " -outputpath Optional output directory to contain the generated cli and cloned repos, default: " +} + +disable_pax_mprotect() +{ + if [[ $(command -v paxctl) ]]; then + paxctl -c -m $1 + fi +} + +get_max_version() +{ + local maxversionhi=0 + local maxversionmid=0 + local maxversionlo=0 + local maxversiontag + local versionrest + local versionhi + local versionmid + local versionlo + local versiontag + local foundmax + + for d in $1/*; do + + if [[ -d $d ]]; then + versionrest=$(basename $d) + versionhi=${versionrest%%.*} + versionrest=${versionrest#*.} + versionmid=${versionrest%%.*} + versionrest=${versionrest#*.} + versionlo=${versionrest%%-*} + versiontag=${versionrest#*-} + if [[ $versiontag == $versionrest ]]; then + versiontag="" + fi + + foundmax=0 + + if [[ $versionhi -gt $maxversionhi ]]; then + foundmax=1 + elif [[ $versionhi -eq $maxversionhi ]]; then + if [[ $versionmid -gt $maxversionmid ]]; then + foundmax=1 + elif [[ $versionmid -eq $maxversionmid ]]; then + if [[ $versionlo -gt $maxversionlo ]]; then + foundmax=1 + elif [[ $versionlo -eq $maxversionlo ]]; then + # tags are used to mark pre-release versions, so a version without a tag + # is newer than a version with one. + if [[ "$versiontag" == "" || $versiontag > $maxversiontag ]]; then + foundmax=1 + fi + fi + fi + fi + + if [[ $foundmax != 0 ]]; then + maxversionhi=$versionhi + maxversionmid=$versionmid + maxversionlo=$versionlo + maxversiontag=$versiontag + fi + fi + done + + echo $maxversionhi.$maxversionmid.$maxversionlo${maxversiontag:+-$maxversiontag} +} + +getrealpath() +{ + if command -v realpath > /dev/null; then + realpath $1 + else + readlink -e $1 + fi +} + +__build_os=Linux +__runtime_id= +__corelib= +__configuration=release +__clangversion= +__outputpath= + +while [[ "${1:-}" != "" ]]; do + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + -h|--help) + usage + exit 1 + ;; + -rid) + shift + __runtime_id=$1 + ;; + -os) + shift + __build_os=$1 + ;; + -debug) + __configuration=debug + ;; + -release) + __configuration=release + ;; + -corelib) + shift + __corelib=`getrealpath $1` + ;; + -seedcli) + shift + __seedclipath=`getrealpath $1` + ;; + -clang) + shift + __clangversion=clang$1 + ;; + -outputpath) + shift + __outputpath=$1 + ;; + *) + echo "Unknown argument to build.sh $1"; exit 1 + esac + shift +done + + +if [[ -z "$__runtime_id" ]]; then + echo "Missing the required -rid argument" + exit 2 +fi + +if [[ -z "$__seedclipath" ]]; then + echo "Missing the required -seedcli argument" + exit 3 +fi + +__build_arch=${__runtime_id#*-} + +if [[ -z "$__outputpath" ]]; then + __outputpath=$__runtime_id/dotnetcli +fi + +if [[ -d "$__outputpath" ]]; then + /bin/rm -r $__outputpath +fi + +mkdir -p $__runtime_id +mkdir -p $__outputpath + +__outputpath=`getrealpath $__outputpath` + +cd $__runtime_id + +cp -r $__seedclipath/* $__outputpath + +__frameworkversion="2.0.0" +__sdkversion="2.0.0" +__fxrversion="2.0.0" + +echo "**** DETECTING VERSIONS IN SEED CLI ****" + +__frameworkversion=`get_max_version $__seedclipath/shared/Microsoft.NETCore.App` +__sdkversion=`get_max_version $__seedclipath/sdk` +__fxrversion=`get_max_version $__seedclipath/host/fxr` + +echo "Framework version: $__frameworkversion" +echo "SDK version: $__sdkversion" +echo "FXR version: $__fxrversion" + +__frameworkpath=$__outputpath/shared/Microsoft.NETCore.App/$__frameworkversion + +echo "**** DETECTING GIT COMMIT HASHES ****" + +# Extract the git commit hashes representig the state of the three repos that +# the seed cli package was built from +__coreclrhash=`strings $__seedclipath/shared/Microsoft.NETCore.App/$__frameworkversion/libcoreclr.so | grep "@(#)" | grep -o "[a-f0-9]\{40\}"` +__corefxhash=`strings $__seedclipath/shared/Microsoft.NETCore.App/$__frameworkversion/System.Native.so | grep "@(#)" | grep -o "[a-f0-9]\{40\}"` +__coresetuphash=`strings $__seedclipath/dotnet | grep "@(#)" | grep -o "[a-f0-9]\{40\}"` +if [[ "$__coresetuphash" == "" ]]; then + __coresetuphash=`strings $__seedclipath/dotnet | grep -o "[a-f0-9]\{40\}"` +fi + +echo "coreclr hash: $__coreclrhash" +echo "corefx hash: $__corefxhash" +echo "core-setup hash: $__coresetuphash" + +# Clone the three repos if they were not cloned yet. If the folders already +# exist, leave them alone. This allows patching the cloned sources as needed + +if [[ ! -d coreclr ]]; then + echo "**** CLONING CORECLR REPOSITORY ****" + git clone https://github.com/dotnet/coreclr.git + cd coreclr + git checkout $__coreclrhash + cd .. +fi + +if [[ ! -d corefx ]]; then + echo "**** CLONING COREFX REPOSITORY ****" + git clone https://github.com/dotnet/corefx.git + cd corefx + git checkout $__corefxhash + cd .. +fi + +if [[ ! -d core-setup ]]; then + echo "**** CLONING CORE-SETUP REPOSITORY ****" + git clone https://github.com/dotnet/core-setup.git + cd core-setup + git checkout $__coresetuphash + cd .. +fi + +echo "**** BUILDING CORE-SETUP NATIVE COMPONENTS ****" +cd core-setup +src/corehost/build.sh --configuration $__configuration --arch "$__build_arch" --hostver "2.0.0" --apphostver "2.0.0" --fxrver "2.0.0" --policyver "2.0.0" --commithash `git rev-parse HEAD` +cd .. + +echo "**** BUILDING CORECLR NATIVE COMPONENTS ****" +cd coreclr +./build.sh $__configuration $__build_arch $__clangversion -skipgenerateversion -skipmanaged -skipmscorlib -skiprestore -skiprestoreoptdata -skipnuget -nopgooptimize 2>&1 | tee coreclr.log +export __coreclrbin=$(cat coreclr.log | sed -n -e 's/^.*Product binaries are available at //p') +cd .. +echo "CoreCLR binaries will be copied from $__coreclrbin" + +echo "**** BUILDING COREFX NATIVE COMPONENTS ****" +corefx/src/Native/build-native.sh $__build_arch $__configuration $__clangversion $__build_os 2>&1 | tee corefx.log +export __corefxbin=$(cat corefx.log | sed -n -e 's/^.*Build files have been written to: //p') +echo "CoreFX binaries will be copied from $__corefxbin" + +echo "**** Copying new binaries to dotnetcli/ ****" + +# First copy the coreclr repo binaries +cp $__coreclrbin/*so $__frameworkpath +cp $__coreclrbin/corerun $__frameworkpath +cp $__coreclrbin/crossgen $__frameworkpath + +# Mark the coreclr executables as allowed to create executable memory mappings +disable_pax_mprotect $__frameworkpath/corerun +disable_pax_mprotect $__frameworkpath/crossgen + +# Now copy the core-setup repo binaries + +if [[ $__fxrversion == 2* ]]; then + cp core-setup/cli/exe/dotnet/dotnet $__outputpath + cp core-setup/cli/exe/dotnet/dotnet $__frameworkpath/corehost + + cp core-setup/cli/dll/libhostpolicy.so $__frameworkpath + cp core-setup/cli/dll/libhostpolicy.so $__outputpath/sdk/$__sdkversion + + cp core-setup/cli/fxr/libhostfxr.so $__frameworkpath + cp core-setup/cli/fxr/libhostfxr.so $__outputpath/host/fxr/$__fxrversion + cp core-setup/cli/fxr/libhostfxr.so $__outputpath/sdk/$__sdkversion +else + cp core-setup/bin/$__runtime_id.$__configuration/corehost/dotnet $__outputpath + cp core-setup/bin/$__runtime_id.$__configuration/corehost/dotnet $__frameworkpath/corehost + + cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostpolicy.so $__frameworkpath + cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostpolicy.so $__outputpath/sdk/$__sdkversion + + cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostfxr.so $__frameworkpath + cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostfxr.so $__outputpath/host/fxr/$__fxrversion + cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostfxr.so $__outputpath/sdk/$__sdkversion +fi + +# Mark the core-setup executables as allowed to create executable memory mappings +disable_pax_mprotect $__outputpath/dotnet +disable_pax_mprotect $__frameworkpath/corehost + +# Finally copy the corefx repo binaries +cp $__corefxbin/**/System.* $__frameworkpath + +# Copy System.Private.CoreLib.dll override from somewhere if requested +if [[ "$__corelib" != "" ]]; then + cp "$__corelib" $__frameworkpath +fi + +# Add the new RID to Microsoft.NETCore.App.deps.json +# Replace the linux-x64 RID in the target, runtimeTarget and runtimes by the new RID +# and add the new RID to the list of runtimes. +echo "**** Adding new rid to Microsoft.NETCore.App.deps.json ****" + +#TODO: add parameter with the parent RID sequence + +sed \ + -e 's/runtime\.linux-x64/runtime.'$__runtime_id'/g' \ + -e 's/runtimes\/linux-x64/runtimes\/'$__runtime_id'/g' \ + -e 's/Version=v\([0-9].[0-9]\)\/linux-x64/Version=v\1\/'$__runtime_id'/g' \ +$__seedclipath/shared/Microsoft.NETCore.App/$__frameworkversion/Microsoft.NETCore.App.deps.json \ +>$__frameworkpath/Microsoft.NETCore.App.deps.json + +# add the new RID to the list of runtimes iff it does not already exist (sed inplace) +__os_dependencies= +if [[ $__build_os == "Linux" ]]; then + __os_dependencies='"linux", "linux-'$__build_arch'", ' +fi +grep -q "\"$__runtime_id\":" $__frameworkpath/Microsoft.NETCore.App.deps.json || \ +sed -i \ + -e 's/"runtimes": {/&\n "'$__runtime_id'": [\n '"$__os_dependencies"'"unix", "unix-'$__build_arch'", "any", "base"\n ],/g' \ +$__frameworkpath/Microsoft.NETCore.App.deps.json + +__crossgentimeout=120 + +function crossgenone(){ + echo $2/crossgen /MissingDependenciesOK /Platform_Assemblies_Paths $2:$3 /in $1 /out $1.ni >$1.log 2>&1 + timeout $__crossgentimeout $2/crossgen /MissingDependenciesOK /Platform_Assemblies_Paths $2:$3 /in $1 /out $1.ni >>$1.log 2>&1 + exitCode=$? + if [ "$exitCode" == "0" ] + then + rm $1.log + mv $1.ni $1 + elif grep -q -e 'The module was expected to contain an assembly manifest' \ + -e 'An attempt was made to load a program with an incorrect format.' \ + -e 'File is PE32' $1.log + then + rm $1.log + echo "$1" >> crossgenskipped + else + echo "$1" >> crossgenretry + fi +} + +# Run an assembly through ildasm ilasm roundtrip to remove x64 crossgen +function uncrossgenone(){ + echo >> $1.log 2>&1 + echo mv $1 $1.x64 >> $1.log 2>&1 + echo $2/ildasm -raweh -out=$1.il $1.x64 "&& \\" >> $1.log 2>&1 + echo $2/ilasm -output=$1 -QUIET -NOLOGO -DEBUG -OPTIMIZE $1.il >> $1.log 2>&1 + + mv $1 $1.x64 + $2/ildasm -raweh -out=$1.il $1.x64 && \ + $2/ilasm -output=$1 -DLL -QUIET -NOLOGO -DEBUG -OPTIMIZE $1.il + exitCode=$? + if [ "$exitCode" == "0" ] + then + rm $1.x64 + rm $1.il + else + echo "$1" >> uncrossgenfails + fi +} + +# if $__build_arch is not x64 then any dll which was crossgened for x64 must be recrossgened for $__build_arch +if [[ "$__build_arch" != "x64" ]] +then + echo "**** Beginning crossgen for $__build_arch target ****" + export -f crossgenone + export __crossgentimeout + + rm -f crossgenretry crossgendlls crossgenskipped uncrossgenfails + + # Assumes System.Private.CoreLib was already crossgened + find $__outputpath -type f -name \*.dll -or -name \*.exe | grep -v System.Private.CoreLib > crossgendlls + + cat crossgendlls | xargs -P 0 -n 1 -I {} bash -c 'crossgenone "$@"' _ {} "$__frameworkpath" "$__outputpath/sdk/$__sdkversion" + + echo + echo "**** Crossgen skipped for non-managed assembly files:" + echo + + touch crossgenskipped + cat crossgenskipped + + echo + echo "**** Crossgen failed for the following dlls:" + echo + + touch crossgenretry + cat crossgenretry + + echo + echo "**** Beginning uncrossgen for failed dlls ****" + echo + export -f uncrossgenone + + rm -f $__coreclrbin/System.Private.CoreLib.dll + ln -s $__corelib $__coreclrbin/System.Private.CoreLib.dll + + cat crossgenretry | xargs -P 0 -n 1 -I {} bash -c 'uncrossgenone "$@"' _ {} "$__coreclrbin" + + rm -f $__coreclrbin/System.Private.CoreLib.dll + + echo + echo "**** Uncrossgen failed for the following dlls:" + echo + touch uncrossgenfails + cat uncrossgenfails +fi + +echo "**** Bootstrap CLI was successfully built ****" + diff --git a/src/SourceBuild/tarball/content/scripts/docker/docker-run.cmd b/src/SourceBuild/tarball/content/scripts/docker/docker-run.cmd new file mode 100644 index 000000000..e7c0d21bb --- /dev/null +++ b/src/SourceBuild/tarball/content/scripts/docker/docker-run.cmd @@ -0,0 +1,47 @@ + +@echo off +setlocal + +set SCRIPT_ROOT=%~dp0 +set REPO_ROOT=%SCRIPT_ROOT%..\..\ + +:arg_loop +set SET_DOCKERFILE= +set SET_DOCKERIMAGE= +if /I "%1" equ "-d" (set SET_DOCKERFILE=1) +if /I "%1" equ "--dockerfile" (set SET_DOCKERFILE=1) +if "%SET_DOCKERFILE%" == "1" ( + echo "1: %1 2: %2" + set DOCKER_FILE=%2 + shift /1 + shift /1 + goto :arg_loop +) +if /I "%1" equ "-i" (set SET_DOCKERIMAGE=1) +if /I "%1" equ "--image" (set SET_DOCKERIMAGE=1) +if "%SET_DOCKERIMAGE%" == "1" ( + set DOCKER_IMAGE=%2 + shift /1 + shift /1 + goto :arg_loop +) + +if "%DOCKER_FILE%" == "" ( + echo Missing required parameter --dockerfile [docker file dir] + exit /b 1 +) +if "%DOCKER_IMAGE%" == "" ( + echo Missing required parameter --image [image name] + exit /b 1 +) + +if EXIST "%DOCKER_FILE%\Dockerfile" ( + docker build -q -f %DOCKER_FILE%\Dockerfile -t %DOCKER_IMAGE% %DOCKER_FILE% +) else ( + echo Error: %DOCKER_FILE%\Dockerfile does not exist + exit /b 1 +) + +docker run -i -t --rm --init -v %REPO_ROOT%:/code -t -w /code %DOCKER_IMAGE% /bin/sh +endlocal + diff --git a/src/SourceBuild/tarball/content/scripts/docker/docker-run.sh b/src/SourceBuild/tarball/content/scripts/docker/docker-run.sh new file mode 100644 index 000000000..93e0382a0 --- /dev/null +++ b/src/SourceBuild/tarball/content/scripts/docker/docker-run.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +DOCKER_FILE="" +DOCKER_IMAGE="" +SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)" +REPO_ROOT="$(cd -P "$SCRIPT_ROOT/../../" && pwd)" + +case $(echo $1 | awk '{print tolower($0)}') in + -d | --dockerfile) + DOCKER_FILE=$2 + ;; + -i | --image) + DOCKER_IMAGE=$2 + ;; + *) + echo "usage: $0 [[-d | --dockerfile] ] | [-i | --image] ]] cmd-to-run" + exit 1 + ;; +esac + +shift +shift + +if [ $DOCKER_FILE ]; then + if [ -d $DOCKER_FILE ]; then + DOCKER_FILE="$DOCKER_FILE/Dockerfile" + fi + + DOCKER_FILE_DIR=$(dirname $DOCKER_FILE) + DOCKER_IMAGE=$(set -x ; docker build -q -f $DOCKER_FILE $DOCKER_FILE_DIR) +fi + +DOCKER_USERADD_AND_SWITCH_CMD="" + +if [ ! $(id -u) = 0 ]; then + DOCKER_USERADD_AND_SWITCH_CMD="useradd -m -u $(id -u) $(id -n -u) && su $(id -n -u) -c " +fi + +ARGS=$(IFS=' ' ; echo $@) +(set -x ; docker run --rm --init -v $REPO_ROOT:/code -t $DOCKER_IMAGE /bin/sh -c "cd /code ; $DOCKER_USERADD_AND_SWITCH_CMD\"$ARGS\"") diff --git a/src/SourceBuild/tarball/content/scripts/generate-readme-table.sh b/src/SourceBuild/tarball/content/scripts/generate-readme-table.sh new file mode 100644 index 000000000..244306d7d --- /dev/null +++ b/src/SourceBuild/tarball/content/scripts/generate-readme-table.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_root="$(cd -P "$( dirname "$0" )" && pwd)" + +branch=master +branch_azdo=$branch + +readme="$script_root/../README.md" + +if [ ! -f "$readme" ]; then + echo "$readme must exist." + exit 1 +fi + +print_rows() { + echo '| OS | *Azure DevOps*
Release |' + echo '| -- | :-- |' + row 'CentOS7.1' 'Production' + row 'CentOS7.1' 'Online' + row 'CentOS7.1' 'Offline' + row 'CentOS7.1' 'Offline Portable' + row 'Debian8.2' 'Production' + row 'Debian8.2' 'Online' + row 'Fedora29' 'Production' + row 'Fedora29' 'Online' + row 'Fedora29' 'Offline' + row 'Fedora29' 'Offline Portable' + row 'OSX' 'Production' + row 'Ubuntu16.04' 'Production' + row 'Windows' 'Production' +} + +raw_print() { + printf '%s' "$1" +} + +row() { + os=$1 + job_type=$2 + display_name=$os + if [ "$job_type" != "Production" ]; then + display_name="$display_name ($job_type)" + fi + printf "| $display_name | " + azdo + end +} + +end() { + printf '\n' +} + +azdo() { + job=$(raw_print $os | awk '{print tolower($0)}' | sed 's/\.//g') + + # Fix case: AzDO has "sticky" casing across build def lifetime, so these names are inconsistent. + # https://dev.azure.com/dnceng/internal/_workitems/edit/98 + case $os in + OSX|Windows) + job=$os + ;; + esac + + job_type_escaped=$(raw_print "$job_type" | sed 's/ /%20/g') + query="?branchName=$branch_azdo&jobname=$job&configuration=$job_type_escaped" + + raw_print "[![Build Status](https://dev.azure.com/dnceng/internal/_apis/build/status/dotnet/source-build/source-build-CI$query)]" + raw_print "(https://dev.azure.com/dnceng/internal/_build/latest?definitionId=114&branchName=$branch_azdo) | " +} + +none() { + raw_print '| ' +} + +cp "$readme" "$readme.old" + +phase=before +while read line; do + if [ "$phase" = before ]; then + echo "$line" + if [ "$line" = '' ]; then + print_rows + phase=skip + fi + elif [ "$phase" = skip ]; then + if [ "$line" = '' ]; then + echo "$line" + phase=after + fi + else + echo "$line" + fi +done < "$readme.old" > "$readme" + +rm "$readme.old" diff --git a/src/SourceBuild/tarball/content/smoke-test.sh b/src/SourceBuild/tarball/content/smoke-test.sh new file mode 100755 index 000000000..c97da1044 --- /dev/null +++ b/src/SourceBuild/tarball/content/smoke-test.sh @@ -0,0 +1,674 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)" +TARBALL_PREFIX=dotnet-sdk- +VERSION_PREFIX=5.0 +# See https://github.com/dotnet/source-build/issues/579, this version +# needs to be compatible with the runtime produced from source-build +DEV_CERTS_VERSION_DEFAULT=5.0.0-preview.3 +__ROOT_REPO=$(sed 's/\r$//' "$SCRIPT_ROOT/artifacts/obj/rootrepo.txt") # remove CR if mounted repo on Windows drive +executingUserHome=${HOME:-} + +export DOTNET_CLI_TELEMETRY_OPTOUT=1 +export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + +# Use uname to determine what the CPU is. +cpuName=$(uname -p) +# Some Linux platforms report unknown for platform, but the arch for machine. +if [[ "$cpuName" == "unknown" ]]; then + cpuName=$(uname -m) +fi + +case $cpuName in + aarch64) + buildArch=arm64 + ;; + amd64|x86_64) + buildArch=x64 + ;; + armv*l) + buildArch=arm + ;; + i686) + buildArch=x86 + ;; + *) + echo "Unknown CPU $cpuName detected, treating it as x64" + buildArch=x64 + ;; +esac + +projectOutput=false +keepProjects=false +dotnetDir="" +configuration="Release" +excludeNonWebTests=false +excludeWebTests=false +excludeWebNoHttpsTests=false +excludeWebHttpsTests=false +excludeLocalTests=false +excludeOnlineTests=false +devCertsVersion="$DEV_CERTS_VERSION_DEFAULT" +testingDir="$SCRIPT_ROOT/testing-smoke" +cliDir="$testingDir/builtCli" +logFile="$testingDir/smoke-test.log" +restoredPackagesDir="$testingDir/packages" +testingHome="$testingDir/home" +archiveRestoredPackages=false +archivedPackagesDir="$testingDir/smoke-test-packages" +smokeTestPrebuilts="$SCRIPT_ROOT/packages/smoke-test-packages" +runningOnline=false +runningHttps=false + +function usage() { + echo "" + echo "usage:" + echo " --dotnetDir the directory from which to run dotnet" + echo " --configuration the configuration being tested (default=Release)" + echo " --targetRid override the target rid to use when needed (e.g. for self-contained publish tests)" + echo " --projectOutput echo dotnet's output to console" + echo " --keepProjects keep projects after tests are complete" + echo " --minimal run minimal set of tests - local sources only, no web" + echo " --excludeNonWebTests don't run tests for non-web projects" + echo " --excludeWebTests don't run tests for web projects" + echo " --excludeWebNoHttpsTests don't run web project tests with --no-https" + echo " --excludeWebHttpsTests don't run web project tests with https using dotnet-dev-certs" + echo " --excludeLocalTests exclude tests that use local sources for nuget packages" + echo " --excludeOnlineTests exclude test that use online sources for nuget packages" + echo " --devCertsVersion use dotnet-dev-certs instead of default $DEV_CERTS_VERSION_DEFAULT" + echo " --prodConBlobFeedUrl override the prodcon blob feed specified in ProdConFeed.txt, removing it if empty" + echo " --archiveRestoredPackages capture all restored packages to $archivedPackagesDir" + echo "environment:" + echo " prodConBlobFeedUrl override the prodcon blob feed specified in ProdConFeed.txt, removing it if empty" + echo "" +} + +while :; do + if [ $# -le 0 ]; then + break + fi + + lowerI="$(echo "$1" | awk '{print tolower($0)}')" + case $lowerI in + '-?'|-h|--help) + usage + exit 0 + ;; + --dotnetdir) + shift + dotnetDir="$1" + ;; + --configuration) + shift + configuration="$1" + ;; + --targetrid) + shift + targetRid="$1" + ;; + --projectoutput) + projectOutput=true + ;; + --keepprojects) + keepProjects=true + ;; + --minimal) + excludeOnlineTests=true + ;; + --excludenonwebtests) + excludeNonWebTests=true + ;; + --excludewebtests) + excludeWebTests=true + ;; + --excludewebnohttpstests) + excludeWebNoHttpsTests=true + ;; + --excludewebhttpstests) + excludeWebHttpsTests=true + ;; + --excludelocaltests) + excludeLocalTests=true + ;; + --excludeonlinetests) + excludeOnlineTests=true + ;; + --devcertsversion) + shift + devCertsVersion="$1" + ;; + --prodconblobfeedurl) + shift + prodConBlobFeedUrl="$1" + ;; + --archiverestoredpackages) + archiveRestoredPackages=true + ;; + *) + echo "Unrecognized argument '$1'" + usage + exit 1 + ;; + esac + + shift +done + +prodConBlobFeedUrl="${prodConBlobFeedUrl-$(cat "$SCRIPT_ROOT/ProdConFeed.txt")}" + +function doCommand() { + lang=$1 + proj=$2 + shift; shift; + + echo "starting language $lang, type $proj" | tee -a smoke-test.log + + dotnetCmd=${dotnetDir}/dotnet + mkdir "${lang}_${proj}" + cd "${lang}_${proj}" + + newArgs="new $proj -lang $lang" + + while :; do + if [ $# -le 0 ]; then + break + fi + case "$1" in + --new-arg) + shift + newArgs="$newArgs $1" + ;; + *) + break + ;; + esac + shift + done + + while :; do + if [ $# -le 0 ]; then + break + fi + + binlogOnlinePart="local" + binlogHttpsPart="nohttps" + if [ "$runningOnline" == "true" ]; then + binlogOnlinePart="online" + fi + if [ "$runningHttps" == "true" ]; then + binlogHttpsPart="https" + fi + + binlogPrefix="$testingDir/${lang}_${proj}_${binlogOnlinePart}_${binlogHttpsPart}_" + binlog="${binlogPrefix}$1.binlog" + echo " running $1" | tee -a "$logFile" + + if [ "$1" == "new" ]; then + if [ "$projectOutput" == "true" ]; then + "${dotnetCmd}" $newArgs --no-restore | tee -a "$logFile" + else + "${dotnetCmd}" $newArgs --no-restore >> "$logFile" 2>&1 + fi + elif [[ "$1" == "run" && "$proj" =~ ^(web|mvc|webapi|razor|blazorwasm|blazorserver)$ ]]; then + # A separate log file that we will over-write all the time. + exitLogFile="$testingDir/exitLogFile" + echo > "$exitLogFile" + # Run an application in the background and redirect its + # stdout+stderr to a separate process (tee). The tee process + # writes its input to 2 files: + # - Either the normal log or stdout + # - A log that's only used to find out when it's safe to kill + # the application. + if [ "$projectOutput" == "true" ]; then + "${dotnetCmd}" $1 2>&1 > >(tee -a "$exitLogFile") & + else + "${dotnetCmd}" $1 2>&1 > >(tee -a "$logFile" "$exitLogFile" >/dev/null) & + fi + webPid=$! + killCommand="pkill -SIGTERM -P $webPid" + echo " waiting up to 30 seconds for web project with pid $webPid..." + echo " to clean up manually after an interactive cancellation, run: $killCommand" + for seconds in $(seq 30); do + if grep 'Application started. Press Ctrl+C to shut down.' "$exitLogFile"; then + echo " app ready for shutdown after $seconds seconds" + break + fi + sleep 1 + done + echo " stopping $webPid" | tee -a "$logFile" + $killCommand + wait $! + echo " terminated with exit code $?" | tee -a "$logFile" + elif [ "$1" == "multi-rid-publish" ]; then + runPublishScenarios() { + "${dotnetCmd}" publish --self-contained false /bl:"${binlogPrefix}publish-fx-dep.binlog" + "${dotnetCmd}" publish --self-contained true -r "$targetRid" /bl:"${binlogPrefix}publish-self-contained-${targetRid}.binlog" + "${dotnetCmd}" publish --self-contained true -r linux-x64 /bl:"${binlogPrefix}publish-self-contained-portable.binlog" + } + if [ "$projectOutput" == "true" ]; then + runPublishScenarios | tee -a "$logFile" + else + runPublishScenarios >> "$logFile" 2>&1 + fi + else + if [ "$projectOutput" == "true" ]; then + "${dotnetCmd}" $1 /bl:"$binlog" | tee -a "$logFile" + else + "${dotnetCmd}" $1 /bl:"$binlog" >> "$logFile" 2>&1 + fi + fi + if [ $? -eq 0 ]; then + echo " $1 succeeded" >> "$logFile" + else + echo " $1 failed with exit code $?" | tee -a "$logFile" + fi + + shift + done + + cd .. + + if [ "$keepProjects" == "false" ]; then + rm -rf "${lang}_${proj}" + fi + + echo "finished language $lang, type $proj" | tee -a smoke-test.log +} + +function setupDevCerts() { + echo "Setting up dotnet-dev-certs $devCertsVersion to generate dev certificate" | tee -a "$logFile" + ( + set -x + "$dotnetDir/dotnet" tool install -g dotnet-dev-certs --version "$devCertsVersion" --add-source https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + export DOTNET_ROOT="$dotnetDir" + "$testingHome/.dotnet/tools/dotnet-dev-certs" https + ) >> "$logFile" 2>&1 +} + +function runAllTests() { + # Run tests for each language and template + if [ "$excludeNonWebTests" == "false" ]; then + doCommand C# console new restore build run multi-rid-publish + doCommand C# classlib new restore build multi-rid-publish + doCommand C# xunit new restore test + doCommand C# mstest new restore test + + doCommand VB console new restore build run multi-rid-publish + doCommand VB classlib new restore build multi-rid-publish + doCommand VB xunit new restore test + doCommand VB mstest new restore test + + doCommand F# console new restore build run multi-rid-publish + doCommand F# classlib new restore build multi-rid-publish + doCommand F# xunit new restore test + doCommand F# mstest new restore test + fi + + if [ "$excludeWebTests" == "false" ]; then + if [ "$excludeWebNoHttpsTests" == "false" ]; then + runningHttps=false + runWebTests --new-arg --no-https + fi + + if [ "$excludeWebHttpsTests" == "false" ]; then + runningHttps=true + setupDevCerts + runWebTests + fi + fi +} + +function runWebTests() { + doCommand C# web "$@" new restore build run multi-rid-publish + doCommand C# mvc "$@" new restore build run multi-rid-publish + doCommand C# webapi "$@" new restore build multi-rid-publish + doCommand C# razor "$@" new restore build run multi-rid-publish + doCommand C# blazorwasm "$@" new restore build run publish + doCommand C# blazorserver "$@" new restore build run publish + + doCommand F# web "$@" new restore build run multi-rid-publish + doCommand F# mvc "$@" new restore build run multi-rid-publish + doCommand F# webapi "$@" new restore build run multi-rid-publish +} + +function runXmlDocTests() { + targetingPacksDir="$dotnetDir/packs/" + echo "Looking for xml docs in targeting packs in $targetingPacksDir" + + netstandardIgnoreList=( + Microsoft.Win32.Primitives.xml + mscorlib.xml + System.AppContext.xml + System.Buffers.xml + System.Collections.Concurrent.xml + System.Collections.NonGeneric.xml + System.Collections.Specialized.xml + System.Collections.xml + System.ComponentModel.Composition.xml + System.ComponentModel.EventBasedAsync.xml + System.ComponentModel.Primitives.xml + System.ComponentModel.TypeConverter.xml + System.ComponentModel.xml + System.Console.xml + System.Core.xml + System.Data.Common.xml + System.Data.xml + System.Diagnostics.Contracts.xml + System.Diagnostics.Debug.xml + System.Diagnostics.FileVersionInfo.xml + System.Diagnostics.Process.xml + System.Diagnostics.StackTrace.xml + System.Diagnostics.TextWriterTraceListener.xml + System.Diagnostics.Tools.xml + System.Diagnostics.TraceSource.xml + System.Diagnostics.Tracing.xml + System.Drawing.Primitives.xml + System.Drawing.xml + System.Dynamic.Runtime.xml + System.Globalization.Calendars.xml + System.Globalization.Extensions.xml + System.Globalization.xml + System.IO.Compression.FileSystem.xml + System.IO.Compression.xml + System.IO.Compression.ZipFile.xml + System.IO.FileSystem.DriveInfo.xml + System.IO.FileSystem.Primitives.xml + System.IO.FileSystem.Watcher.xml + System.IO.FileSystem.xml + System.IO.IsolatedStorage.xml + System.IO.MemoryMappedFiles.xml + System.IO.Pipes.xml + System.IO.UnmanagedMemoryStream.xml + System.IO.xml + System.Linq.Expressions.xml + System.Linq.Parallel.xml + System.Linq.Queryable.xml + System.Linq.xml + System.Memory.xml + System.Net.Http.xml + System.Net.NameResolution.xml + System.Net.NetworkInformation.xml + System.Net.Ping.xml + System.Net.Primitives.xml + System.Net.Requests.xml + System.Net.Security.xml + System.Net.Sockets.xml + System.Net.WebHeaderCollection.xml + System.Net.WebSockets.Client.xml + System.Net.WebSockets.xml + System.Net.xml + System.Numerics.Vectors.xml + System.Numerics.xml + System.ObjectModel.xml + System.Reflection.DispatchProxy.xml + System.Reflection.Emit.ILGeneration.xml + System.Reflection.Emit.Lightweight.xml + System.Reflection.Emit.xml + System.Reflection.Extensions.xml + System.Reflection.Primitives.xml + System.Reflection.xml + System.Resources.Reader.xml + System.Resources.ResourceManager.xml + System.Resources.Writer.xml + System.Runtime.CompilerServices.VisualC.xml + System.Runtime.Extensions.xml + System.Runtime.Handles.xml + System.Runtime.InteropServices.RuntimeInformation.xml + System.Runtime.InteropServices.xml + System.Runtime.Numerics.xml + System.Runtime.Serialization.Formatters.xml + System.Runtime.Serialization.Json.xml + System.Runtime.Serialization.Primitives.xml + System.Runtime.Serialization.xml + System.Runtime.Serialization.Xml.xml + System.Runtime.xml + System.Security.Claims.xml + System.Security.Cryptography.Algorithms.xml + System.Security.Cryptography.Csp.xml + System.Security.Cryptography.Encoding.xml + System.Security.Cryptography.Primitives.xml + System.Security.Cryptography.X509Certificates.xml + System.Security.Principal.xml + System.Security.SecureString.xml + System.ServiceModel.Web.xml + System.Text.Encoding.Extensions.xml + System.Text.Encoding.xml + System.Text.RegularExpressions.xml + System.Threading.Overlapped.xml + System.Threading.Tasks.Extensions.xml + System.Threading.Tasks.Parallel.xml + System.Threading.Tasks.xml + System.Threading.ThreadPool.xml + System.Threading.Thread.xml + System.Threading.Timer.xml + System.Threading.xml + System.Transactions.xml + System.ValueTuple.xml + System.Web.xml + System.Windows.xml + System.xml + System.Xml.Linq.xml + System.Xml.ReaderWriter.xml + System.Xml.Serialization.xml + System.Xml.XDocument.xml + System.Xml.xml + System.Xml.XmlDocument.xml + System.Xml.XmlSerializer.xml + System.Xml.XPath.XDocument.xml + System.Xml.XPath.xml + ) + + netcoreappIgnoreList=( + Microsoft.VisualBasic.xml + netstandard.xml + System.AppContext.xml + System.Buffers.xml + System.ComponentModel.DataAnnotations.xml + System.Configuration.xml + System.Core.xml + System.Data.DataSetExtensions.xml + System.Data.xml + System.Diagnostics.Debug.xml + System.Diagnostics.Tools.xml + System.Drawing.xml + System.Dynamic.Runtime.xml + System.Globalization.Calendars.xml + System.Globalization.Extensions.xml + System.Globalization.xml + System.IO.Compression.FileSystem.xml + System.IO.FileSystem.Primitives.xml + System.IO.UnmanagedMemoryStream.xml + System.IO.xml + System.Net.xml + System.Numerics.xml + System.Reflection.Extensions.xml + System.Reflection.xml + System.Resources.Reader.xml + System.Resources.ResourceManager.xml + System.Runtime.Extensions.xml + System.Runtime.Handles.xml + System.Runtime.Serialization.xml + System.Security.Principal.xml + System.Security.SecureString.xml + System.Security.xml + System.ServiceModel.Web.xml + System.ServiceProcess.xml + System.Text.Encoding.xml + System.Threading.Tasks.Extensions.xml + System.Threading.Tasks.xml + System.Threading.Timer.xml + System.Transactions.xml + System.ValueTuple.xml + System.Web.xml + System.Windows.xml + System.xml + System.Xml.Linq.xml + System.Xml.Serialization.xml + System.Xml.xml + System.Xml.XmlDocument.xml + ) + + error=0 + while IFS= read -r -d '' dllFile; do + xmlDocFile=${dllFile%.*}.xml + skip=0 + if [[ "$xmlDocFile" == *"/packs/Microsoft.NETCore.App.Ref"* ]]; then + xmlFileBasename=$(basename "$xmlDocFile") + for ignoreItem in "${netcoreappIgnoreList[@]}"; do + if [[ "$ignoreItem" == "$xmlFileBasename" ]]; then + skip=1; + break + fi + done + fi + if [[ "$xmlDocFile" == *"/packs/NETStandard.Library.Ref"* ]]; then + xmlFileBasename=$(basename "$xmlDocFile") + for ignoreItem in "${netstandardIgnoreList[@]}"; do + if [[ "$ignoreItem" == "$xmlFileBasename" ]]; then + skip=1; + break + fi + done + fi + if [[ $skip == 0 ]] && [[ ! -f "$xmlDocFile" ]]; then + error=1 + echo "error: missing $xmlDocFile" + fi + done < <(find "$targetingPacksDir" -name '*.dll' -print0) + + if [[ $error != 0 ]]; then + echo "error: Missing xml documents" + exit 1 + else + echo "All expected xml docs are present" + fi +} + +function resetCaches() { + rm -rf "$testingHome" + mkdir "$testingHome" + + HOME="$testingHome" + + # clean restore path + rm -rf "$restoredPackagesDir" + + # Copy NuGet plugins if running user has HOME and we have auth. In particular, the auth plugin. + if [ "${internalPackageFeedPat:-}" ] && [ "${executingUserHome:-}" ]; then + cp -r "$executingUserHome/.nuget/" "$HOME/.nuget/" || : + fi +} + +function setupProdConFeed() { + if [ "$prodConBlobFeedUrl" ]; then + sed -i.bakProdCon "s|PRODUCT_CONTRUCTION_PACKAGES|$prodConBlobFeedUrl|g" "$testingDir/NuGet.Config" + else + sed -i.bakProdCon "/PRODUCT_CONTRUCTION_PACKAGES/d" "$testingDir/NuGet.Config" + fi +} + +function setupSmokeTestFeed() { + # Setup smoke-test-packages if they exist + if [ -e "$smokeTestPrebuilts" ]; then + sed -i.bakSmokeTestFeed "s|SMOKE_TEST_PACKAGE_FEED|$smokeTestPrebuilts|g" "$testingDir/NuGet.Config" + else + sed -i.bakSmokeTestFeed "/SMOKE_TEST_PACKAGE_FEED/d" "$testingDir/NuGet.Config" + fi +} + +function copyRestoredPackages() { + if [ "$archiveRestoredPackages" == "true" ]; then + mkdir -p "$archivedPackagesDir" + cp -rf "$restoredPackagesDir"/* "$archivedPackagesDir" + fi +} + +echo "RID to test: ${targetRid?not specified. Use ./build.sh --run-smoke-test to detect RID, or specify manually.}" + +if [ "$__ROOT_REPO" != "known-good" ]; then + echo "Skipping smoke-tests since cli was not built"; + exit +fi + +# Clean up and create directory +if [ -e "$testingDir" ]; then + read -p "testing-smoke directory exists, remove it? [Y]es / [n]o" -n 1 -r + echo + if [[ $REPLY == "" || $REPLY == " " || $REPLY =~ ^[Yy]$ ]]; then + rm -rf "$testingDir" + fi +fi + +mkdir -p "$testingDir" +cd "$testingDir" + +# Create blank Directory.Build files to avoid traversing to source-build infra. +echo "" | tee Directory.Build.props > Directory.Build.targets + +# Unzip dotnet if the dotnetDir is not specified +if [ "$dotnetDir" == "" ]; then + OUTPUT_DIR="$SCRIPT_ROOT/artifacts/$buildArch/$configuration/" + DOTNET_TARBALL="$(ls "${OUTPUT_DIR}${TARBALL_PREFIX}${VERSION_PREFIX}"*)" + + mkdir -p "$cliDir" + tar xzf "$DOTNET_TARBALL" -C "$cliDir" + dotnetDir="$cliDir" +else + if ! [[ "$dotnetDir" = /* ]]; then + dotnetDir="$SCRIPT_ROOT/$dotnetDir" + fi +fi + +echo SDK under test is: +"$dotnetDir/dotnet" --info + +# setup restore path +export NUGET_PACKAGES="$restoredPackagesDir" +SOURCE_BUILT_PKGS_PATH="$SCRIPT_ROOT/artifacts/obj/$buildArch/$configuration/blob-feed/packages/" +export DOTNET_ROOT="$dotnetDir" +# OSX also requires DOTNET_ROOT to be on the PATH +if [ "$(uname)" == 'Darwin' ]; then + export PATH="$dotnetDir:$PATH" +fi + +# Run all tests, local restore sources first, online restore sources second +if [ "$excludeLocalTests" == "false" ]; then + resetCaches + runningOnline=false + # Setup NuGet.Config with local restore source + if [ -e "$SCRIPT_ROOT/smoke-testNuGet.Config" ]; then + cp "$SCRIPT_ROOT/smoke-testNuGet.Config" "$testingDir/NuGet.Config" + sed -i.bak "s|SOURCE_BUILT_PACKAGES|$SOURCE_BUILT_PKGS_PATH|g" "$testingDir/NuGet.Config" + setupProdConFeed + setupSmokeTestFeed + echo "$testingDir/NuGet.Config Contents:" + cat "$testingDir/NuGet.Config" + fi + echo "RUN ALL TESTS - LOCAL RESTORE SOURCE" + runAllTests + copyRestoredPackages + echo "LOCAL RESTORE SOURCE - ALL TESTS PASSED!" +fi + +if [ "$excludeOnlineTests" == "false" ]; then + resetCaches + runningOnline=true + # Setup NuGet.Config to use online restore sources + if [ -e "$SCRIPT_ROOT/smoke-testNuGet.Config" ]; then + cp "$SCRIPT_ROOT/smoke-testNuGet.Config" "$testingDir/NuGet.Config" + sed -i.bak "/SOURCE_BUILT_PACKAGES/d" "$testingDir/NuGet.Config" + setupProdConFeed + setupSmokeTestFeed + echo "$testingDir/NuGet.Config Contents:" + cat "$testingDir/NuGet.Config" + fi + echo "RUN ALL TESTS - ONLINE RESTORE SOURCE" + runAllTests + copyRestoredPackages + echo "ONLINE RESTORE SOURCE - ALL TESTS PASSED!" +fi + +runXmlDocTests + +echo "ALL TESTS PASSED!" diff --git a/src/SourceBuild/tarball/content/smoke-testNuGet.Config b/src/SourceBuild/tarball/content/smoke-testNuGet.Config new file mode 100644 index 000000000..07754412d --- /dev/null +++ b/src/SourceBuild/tarball/content/smoke-testNuGet.Config @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/Directory.Build.props b/src/SourceBuild/tarball/content/tools-local/Directory.Build.props new file mode 100644 index 000000000..bffd69d62 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/Directory.Build.props @@ -0,0 +1,13 @@ + + + + + true + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/acquire-darc/acquire-darc.proj b/src/SourceBuild/tarball/content/tools-local/acquire-darc/acquire-darc.proj new file mode 100644 index 000000000..054a65119 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/acquire-darc/acquire-darc.proj @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/generate-graphviz/generate-graphviz.proj b/src/SourceBuild/tarball/content/tools-local/generate-graphviz/generate-graphviz.proj new file mode 100644 index 000000000..888cbd9c6 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/generate-graphviz/generate-graphviz.proj @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + $(BaseIntermediatePath)graphviz.dot + $(BaseIntermediatePath)graphviz.png + digraph { +graph [ dpi = 150 ] +@(RepoLink -> '%(Text)') +} + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/init-build.proj b/src/SourceBuild/tarball/content/tools-local/init-build.proj new file mode 100644 index 000000000..5213830c6 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/init-build.proj @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + $(ReferencePackagesDir)%3B$(PrebuiltPackagesPath) + + + + + + + + + + + + + + + + + + + + + + + + + $(IntermediatePath)PackageVersions.props + + + + + + +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/prebuilt-baseline-offline.xml b/src/SourceBuild/tarball/content/tools-local/prebuilt-baseline-offline.xml new file mode 100644 index 000000000..cca3774b3 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/prebuilt-baseline-offline.xml @@ -0,0 +1,46 @@ + + centos.7-x64 + + src/ApplicationInsights-dotnet.53b80940842204f78708a538628288ff5d741a1d/ + src/arcade.6eec4404c2df5bfa46e5da52383c881c5cca3a9f/ + src/xdt.c01a538851a8ab1a1fbeb2e6243f391fff7587b4/ + src/dotnet-aspnetcore.2670c128d522473e146ff9f8159bfffdfe694cd9/ + src/cliCommandLineParser.0e89c2116ad28e404ba56c14d1c3f938caa25a01/ + src/command-line-api.afd010ba8cb3cbd714c734465d1a5de1ee133e2d/ + src/common.6e37cdfe96ac8b06a923242120169fafacd720e6/ + src/cssparser.d6d86bcd8c162b1ae22ef00955ff748d028dd0ee/ + src/diagnostics.47296ca69bb66180c132f3b16667f904dfc7c6c7/ + src/fsharp.da6be68280c89131cdba2045525b80890401defd/ + src/Humanizer.b30550eed103a6970d8465fe7c5c16300b70be81/ + src/dotnet-installer.71365b4d424b0860ceb9ad8ee47f7cd08c55c362/ + src/known-good-tests./ + src/known-good./ + src/linker.25604250cf2663aed6630e305cc0538aeebda80a/ + src/msbuild.39993bd9d02917993b6147a9973dc3aa6e9b00c7/ + src/netcorecli-fsc/ + src/Newtonsoft.Json.cac0690ad133c5e166ce5642dc71175791404fad/ + src/Newtonsoft.Json.e43dae94c26f0c30e9095327a3a9eac87193923d/ + src/NuGet.Client.830c8be45dbbccd411ecf6080abff0c2c98079cf/ + src/package-source-build/ + src/roslyn-analyzers.77b259353aa44ec1510951a75c6e7ed6e662a001/ + src/roslyn.9ed4b774d20940880de8df1ca8b07508aa01c8cd/ + src/dotnet-runtime.cb5f173b9696d9d00a544b953d95190ab3b56df2/ + src/dotnet-runtime.cb5f173b9696d9d00a544b953d95190ab3b56df2/ + src/dotnet-sdk.ef14c79a16171496e2d972edd9a7874d596f624d/ + src/sourcelink.f175b06862f889474b689a57527e489101c774cc/ + src/symreader.f8a3ba85aed339fb8d08ca26f3876b28c32d58ee/ + src/templating.568c09b99075179607c572f7b9f3ad3d706f1fc3/ + src/test-templates.956e14dedd3a3ac981b320d66c6d389387a2954a/ + src/vstest.212656d7b384a506aa714999f678b3ace82d114e/ + src/xliff-tasks.a52f3d7fb58470749ee4035fbbcb7e63c78b0459/ + Tools/ + tools-local/tasks/ + artifacts/obj/ + + + + + + + + \ No newline at end of file diff --git a/src/SourceBuild/tarball/content/tools-local/prebuilt-baseline-online.xml b/src/SourceBuild/tarball/content/tools-local/prebuilt-baseline-online.xml new file mode 100755 index 000000000..03370c501 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/prebuilt-baseline-online.xml @@ -0,0 +1,457 @@ + + centos.7-x64 + + artifacts/src/ApplicationInsights-dotnet.53b80940842204f78708a538628288ff5d741a1d/ + artifacts/src/arcade.6eec4404c2df5bfa46e5da52383c881c5cca3a9f/ + artifacts/src/xdt.c01a538851a8ab1a1fbeb2e6243f391fff7587b4/ + artifacts/src/dotnet-aspnetcore.b7a2ec8c7ed6b48857af0a69688a73e8c14fe6cb/ + artifacts/src/CliCommandLineParser.0e89c2116ad28e404ba56c14d1c3f938caa25a01/ + artifacts/src/command-line-api.6ddde11ed45e3f4b9d80c97670f347dbfda15c3f/ + artifacts/src/common.6e37cdfe96ac8b06a923242120169fafacd720e6/ + artifacts/src/cssparser.d6d86bcd8c162b1ae22ef00955ff748d028dd0ee/ + artifacts/src/diagnostics.47296ca69bb66180c132f3b16667f904dfc7c6c7/ + artifacts/src/fsharp.7ce7132f1459095e635194d09d6f73265352029a/ + artifacts/src/Humanizer.b30550eed103a6970d8465fe7c5c16300b70be81/ + artifacts/src/dotnet-installer.db7cc87d512335808e3806067f2bf9b31379e1db/ + artifacts/src/known-good-tests./ + artifacts/src/known-good./ + artifacts/src/linker.c43f981eec336c1dc4fd0ead84b5e09db9377d9e/ + artifacts/src/msbuild.5e4b48a27efce55a613664b58d353ab4c8d1f6c1/ + src/netcorecli-fsc/ + artifacts/src/Newtonsoft.Json.cac0690ad133c5e166ce5642dc71175791404fad/ + artifacts/src/Newtonsoft.Json.e43dae94c26f0c30e9095327a3a9eac87193923d/ + artifacts/src/nuget.client.d525b0e670f3b6cbd5c73a35f04730a9f658c852/ + src/package-source-build/ + artifacts/src/roslyn-analyzers.ce71b27be743710012c0460071da188b2f05959c/ + artifacts/src/roslyn.59eedc33d35754759994155ea2f4e1012a9951e3/ + artifacts/src/dotnet-runtime.2f740adc1457e8a28c1c072993b66f515977eb51/ + artifacts/src/dotnet-runtime.2f740adc1457e8a28c1c072993b66f515977eb51/ + artifacts/src/dotnet-sdk.51369266643769f9f0c1184e89715cd1045126d0/ + artifacts/src/sourcelink.f175b06862f889474b689a57527e489101c774cc/ + artifacts/src/symreader.f8a3ba85aed339fb8d08ca26f3876b28c32d58ee/ + artifacts/src/templating.8470ff317250d761c72f920b8ea1c0700b230eb3/ + artifacts/src/test-templates.956e14dedd3a3ac981b320d66c6d389387a2954a/ + artifacts/src/vstest.99b911a57a02fc5d2eeef23e9ab8cbea4505678b/ + artifacts/src/xliff-tasks.a52f3d7fb58470749ee4035fbbcb7e63c78b0459/ + Tools/ + tools-local/tasks/ + artifacts/obj/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Directory.Build.props b/src/SourceBuild/tarball/content/tools-local/tasks/Directory.Build.props new file mode 100644 index 000000000..65f8365de --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Directory.Build.props @@ -0,0 +1,29 @@ + + + + + + AnyCPU + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogFileEntry.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogFileEntry.cs new file mode 100644 index 000000000..268d50fc5 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogFileEntry.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + internal class CatalogFileEntry + { + const string ElementName = "File"; + + internal string Path { get; set; } + internal byte[] OriginalHash { get; set; } + internal byte[] PoisonedHash { get; set; } + + public XElement ToXml() => new XElement(ElementName, + new XAttribute(nameof(Path), Path), + new XAttribute(nameof(OriginalHash), OriginalHash.ToHexString()), + PoisonedHash == null ? null : new XAttribute(nameof(PoisonedHash), PoisonedHash.ToHexString()) + ); + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogPackageEntry.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogPackageEntry.cs new file mode 100644 index 000000000..e52e1aaa2 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogPackageEntry.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + internal class CatalogPackageEntry + { + const string ElementName = "Package"; + + internal string Path { get; set; } + internal string Id { get; set; } + internal string Version { get; set; } + internal byte[] OriginalHash { get; set; } + internal byte[] PoisonedHash { get; set; } + internal List Files { get; } + + public CatalogPackageEntry() + { + this.Files = new List(); + } + + public XElement ToXml() => new XElement(ElementName, + new XAttribute(nameof(Path), Path), + new XAttribute(nameof(Id), Id), + new XAttribute(nameof(Version), Version), + new XAttribute(nameof(OriginalHash), OriginalHash.ToHexString()), + PoisonedHash == null ? null : new XAttribute(nameof(PoisonedHash), PoisonedHash.ToHexString()), + Files.Select(f => f.ToXml()) + ); + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs new file mode 100644 index 000000000..3fad37c47 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs @@ -0,0 +1,384 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Mono.Cecil; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Xml; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + public class CheckForPoison : Task + { + /// + /// The files to check for poison and/or hash matches. Zips and + /// nupkgs will be extracted and checked recursively. + /// %(Identity): Path to the initial set of files. + /// + [Required] + public ITaskItem[] FilesToCheck { get; set; } + + /// + /// The output path for an XML poison report, if desired. + /// + public string PoisonReportOutputFilePath { get; set; } + + /// + /// The path of a previously-generated file hash catalog, if + /// hash checked is desired. If not, only assembly attributes + /// and package marker file checked will be done. + /// + public string HashCatalogFilePath { get; set; } + + /// + /// The marker file name to check for in poisoned nupkg files. + /// + public string MarkerFileName { get; set; } + + /// + /// If true, fails the build if any poisoned files are found. + /// + public bool FailOnPoisonFound { get; set; } + + /// + /// Use this directory instead of the system temp directory for staging. + /// Intended for Linux systems with limited /tmp space, like Azure VMs. + /// + public string OverrideTempPath { get; set; } + + private static readonly string[] ZipFileExtensions = + { + ".zip", + ".nupkg", + }; + + private static readonly string[] TarFileExtensions = + { + ".tar", + }; + + private static readonly string[] TarGzFileExtensions = + { + ".tgz", + ".tar.gz", + }; + + private static readonly string[] FileNamesToSkip = + { + "_._", + "-.-", + ".bowerrc", + ".editorconfig", + ".gitignore", + ".gitkeep", + ".rels", + "LICENSE", + "prefercliruntime", + "RunCsc", + "RunVbc", + }; + + private static readonly string[] FileExtensionsToSkip = + { + ".config", + ".cs", + ".cshtml", + ".csproj", + ".css", + ".db", + ".eot", + ".fs", + ".fsproj", + ".html", + ".ico", + ".js", + ".json", + ".map", + ".md", + ".nuspec", + ".png", + ".props", + ".psmdcp", + ".rtf", + ".scss", + ".svg", + ".targets", + ".ts", + ".ttf", + ".txt", + ".vb", + ".vbproj", + ".woff", + ".woff2", + ".xaml", + ".xml", + }; + + private const string PoisonMarker = "POISONED"; + + public override bool Execute() + { + IEnumerable poisons = GetPoisonedFiles(FilesToCheck.Select(f => f.ItemSpec), HashCatalogFilePath, MarkerFileName); + + // if we should write out the poison report, do that + if (!string.IsNullOrWhiteSpace(PoisonReportOutputFilePath)) + { + File.WriteAllText(PoisonReportOutputFilePath, (new XElement("PrebuiltLeakReport", poisons.Select(p => p.ToXml()))).ToString()); + } + + if (FailOnPoisonFound && poisons.Count() > 0) + { + Log.LogError($"Forced build error: {poisons.Count()} marked files leaked to output. See complete report '{PoisonReportOutputFilePath}' for details."); + return false; + } + else if (poisons.Count() > 0) + { + Log.LogWarning($"{poisons.Count()} marked files leaked to output. See complete report '{PoisonReportOutputFilePath}' for details."); + } + else + { + Log.LogError($"No leaked files found in output. Either something is broken or it is the future and we have fixed all leaks - please verify and remove this error if so (and default {nameof(FailOnPoisonFound)} to true)."); + return false; + } + + return !Log.HasLoggedErrors; + } + + /// + /// Internal helper to allow other tasks to check for poisoned files. + /// + /// Initial queue of candidate files (will be cleared when done) + /// File path to the file hash catalog + /// Marker file name to check for in poisoned nupkgs + /// List of poisoned packages and files found and reasons for each + internal IEnumerable GetPoisonedFiles(IEnumerable initialCandidates, string catalogedPackagesFilePath, string markerFileName) + { + IEnumerable catalogedPackages = ReadCatalog(catalogedPackagesFilePath); + var poisons = new List(); + var candidateQueue = new Queue(initialCandidates); + // avoid collisions between nupkgs with the same name + var dirCounter = 0; + if (!string.IsNullOrWhiteSpace(OverrideTempPath)) + { + Directory.CreateDirectory(OverrideTempPath); + } + var tempDirName = Path.GetRandomFileName(); + var tempDir = Directory.CreateDirectory(Path.Combine(OverrideTempPath ?? Path.GetTempPath(), tempDirName)); + + while (candidateQueue.Any()) + { + var checking = candidateQueue.Dequeue(); + + // if this is a zip or NuPkg, extract it, check for the poison marker, and + // add its contents to the list to be checked. + if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => checking.ToLowerInvariant().EndsWith(e))) + { + var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetRandomFileName(), Path.GetFileNameWithoutExtension(checking) + "." + (++dirCounter).ToString()); + PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, checking, markerFileName, tempCheckingDir, candidateQueue); + if (result != null) + { + poisons.Add(result); + } + } + else + { + PoisonedFileEntry result = CheckSingleFile(catalogedPackages, tempDir.FullName, checking); + if (result != null) + { + poisons.Add(result); + } + } + } + + tempDir.Delete(true); + + return poisons; + } + + private static PoisonedFileEntry CheckSingleFile(IEnumerable catalogedPackages, string rootPath, string fileToCheck) + { + // skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as + // file types that we never care about - text files, .gitconfig, etc. + if (FileNamesToSkip.Any(f => Path.GetFileName(fileToCheck).ToLowerInvariant() == f.ToLowerInvariant()) || + FileExtensionsToSkip.Any(e => Path.GetExtension(fileToCheck).ToLowerInvariant() == e.ToLowerInvariant())) + { + return null; + } + + var poisonEntry = new PoisonedFileEntry(); + poisonEntry.Path = Utility.MakeRelativePath(fileToCheck, rootPath); + + // There seems to be some weird issues with using file streams both for hashing and assembly loading. + // Copy everything into a memory stream to avoid these problems. + var memStream = new MemoryStream(); + using (var stream = File.OpenRead(fileToCheck)) + { + stream.CopyTo(memStream); + } + + memStream.Seek(0, SeekOrigin.Begin); + using (var sha = SHA256.Create()) + { + poisonEntry.Hash = sha.ComputeHash(memStream); + } + + foreach (var p in catalogedPackages) + { + // This hash can match either the original hash (we couldn't poison the file, or redownloaded it) or + // the poisoned hash (the obvious failure case of a poisoned file leaked). + foreach (var matchingCatalogedFile in p.Files.Where(f => f.OriginalHash.SequenceEqual(poisonEntry.Hash) || (f.PoisonedHash?.SequenceEqual(poisonEntry.Hash) ?? false))) + { + poisonEntry.Type |= PoisonType.Hash; + var match = new PoisonMatch + { + File = matchingCatalogedFile.Path, + Package = p.Path, + PackageId = p.Id, + PackageVersion = p.Version, + }; + poisonEntry.Matches.Add(match); + } + } + + try + { + memStream.Seek(0, SeekOrigin.Begin); + using (var asm = AssemblyDefinition.ReadAssembly(memStream)) + { + foreach (var a in asm.CustomAttributes) + { + foreach (var ca in a.ConstructorArguments) + { + if (ca.Type.Name == asm.MainModule.TypeSystem.String.Name) + { + if (ca.Value.ToString().Contains(PoisonMarker)) + { + poisonEntry.Type |= PoisonType.AssemblyAttribute; + } + } + } + } + } + } + catch + { + // this is fine, it's just not an assembly. + } + + return poisonEntry.Type != PoisonType.None ? poisonEntry : null; + } + + private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable catalogedPackages, string zipToCheck, string markerFileName, string tempDir, Queue futureFilesToCheck) + { + var poisonEntry = new PoisonedFileEntry(); + poisonEntry.Path = zipToCheck; + + using (var sha = SHA256.Create()) + using (var stream = File.OpenRead(zipToCheck)) + { + poisonEntry.Hash = sha.ComputeHash(stream); + } + + // first check for a matching poisoned or non-poisoned hash match: + // - non-poisoned is a potential error where the package was redownloaded. + // - poisoned is a use of a local package we were not expecting. + foreach (var matchingCatalogedPackage in catalogedPackages.Where(c => c.OriginalHash.SequenceEqual(poisonEntry.Hash) || (c.PoisonedHash?.SequenceEqual(poisonEntry.Hash) ?? false))) + { + poisonEntry.Type |= PoisonType.Hash; + var match = new PoisonMatch + { + Package = matchingCatalogedPackage.Path, + PackageId = matchingCatalogedPackage.Id, + PackageVersion = matchingCatalogedPackage.Version, + }; + poisonEntry.Matches.Add(match); + } + + // now extract and look for the marker file + if (ZipFileExtensions.Any(e => zipToCheck.ToLowerInvariant().EndsWith(e))) + { + ZipFile.ExtractToDirectory(zipToCheck, tempDir, true); + } + else if (TarFileExtensions.Any(e => zipToCheck.ToLowerInvariant().EndsWith(e))) + { + Directory.CreateDirectory(tempDir); + var psi = new ProcessStartInfo("tar", $"xf {zipToCheck} -C {tempDir}"); + Process.Start(psi).WaitForExit(); + } + else if (TarGzFileExtensions.Any(e => zipToCheck.ToLowerInvariant().EndsWith(e))) + { + Directory.CreateDirectory(tempDir); + var psi = new ProcessStartInfo("tar", $"xzf {zipToCheck} -C {tempDir}"); + Process.Start(psi).WaitForExit(); + } + else + { + throw new ArgumentOutOfRangeException($"Don't know how to decompress {zipToCheck}"); + } + + if (!string.IsNullOrWhiteSpace(markerFileName) && File.Exists(Path.Combine(tempDir, markerFileName))) + { + poisonEntry.Type |= PoisonType.NupkgFile; + } + + foreach (var child in Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories)) + { + // also add anything in this zip/package for checking + futureFilesToCheck.Enqueue(child); + } + + return poisonEntry.Type != PoisonType.None ? poisonEntry : null; + } + + private static IEnumerable ReadCatalog(string hashCatalogFilePath) + { + // catalog is optional, we can also just check assembly properties or nupkg marker files + if (string.IsNullOrWhiteSpace(hashCatalogFilePath)) + { + return Enumerable.Empty(); + } + + var doc = new XmlDocument(); + using (var stream = File.OpenRead(hashCatalogFilePath)) + { + doc.Load(stream); + } + var packages = new List(); + var catalog = doc.FirstChild; + foreach (XmlElement p in catalog.ChildNodes) + { + var package = new CatalogPackageEntry + { + Id = p.Attributes["Id"].Value, + Version = p.Attributes["Version"].Value, + OriginalHash = p.Attributes["OriginalHash"].Value.ToBytes(), + PoisonedHash = p.Attributes["PoisonedHash"]?.Value?.ToBytes(), + Path = p.Attributes["Path"].Value, + }; + packages.Add(package); + foreach (XmlNode f in p.ChildNodes) + { + var fEntry = new CatalogFileEntry + { + OriginalHash = f.Attributes["OriginalHash"].Value.ToBytes(), + PoisonedHash = f.Attributes["PoisonedHash"]?.Value?.ToBytes(), + Path = f.Attributes["Path"].Value, + }; + package.Files.Add(fEntry); + } + } + return packages; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/MarkAndCatalogPackages.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/MarkAndCatalogPackages.cs new file mode 100644 index 000000000..0ef1ed0af --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/MarkAndCatalogPackages.cs @@ -0,0 +1,218 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Mono.Cecil; +using Mono.Collections.Generic; +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using System.Text; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + public class MarkAndCatalogPackages : Task + { + private const string CatalogElementName = "HashCatalog"; + private const string PoisonMarker = "POISONED by DotNetSourceBuild - Should not ship"; + + private readonly Type[] AssemblyPropertiesToReplace = new Type[] { + typeof(AssemblyProductAttribute), + typeof(AssemblyInformationalVersionAttribute), + typeof(AssemblyDescriptionAttribute), + typeof(AssemblyTitleAttribute) + }; + + /// + /// The name of the XML file to write the hash catalog out to, + /// for later checking build output against. This is optional - + /// if not used, assemblies will still be poisoned in their attributes. + /// + public string CatalogOutputFilePath { get; set; } + + /// + /// The name of the marker file to drop in the nupkgs. This can vary + /// with the packages, so you can use ".prebuilt" for one set of packages + /// and ".source-built" for another if you would like to tell the difference + /// between two sets of poisoned packages. + /// + public string MarkerFileName { get; set; } + + /// + /// The packages to poison and/or catalog: + /// %(Identity): Path to the nupkg. + /// + [Required] + public ITaskItem[] PackagesToMark { get; set; } + + /// + /// Use this directory instead of the system temp directory for staging. + /// Intended for Linux systems with limited /tmp space, like Azure VMs. + /// + public string OverrideTempPath { get; set; } + + public override bool Execute() + { + var tempDirName = Path.GetRandomFileName(); + if (!string.IsNullOrWhiteSpace(OverrideTempPath)) + { + Directory.CreateDirectory(OverrideTempPath); + } + var tempDir = Directory.CreateDirectory(Path.Combine(OverrideTempPath ?? Path.GetTempPath(), tempDirName)); + + var packageEntries = new List(); + + using (var sha = SHA256.Create()) + { + foreach (var p in PackagesToMark) + { + var packageEntry = new CatalogPackageEntry(); + packageEntries.Add(packageEntry); + packageEntry.Path = p.ItemSpec; + using (var stream = File.OpenRead(p.ItemSpec)) + { + packageEntry.OriginalHash = sha.ComputeHash(stream); + } + var packageIdentity = ReadNuGetPackageInfos.ReadIdentity(p.ItemSpec); + packageEntry.Id = packageIdentity.Id; + packageEntry.Version = packageIdentity.Version.OriginalVersion; + var packageTempPath = Path.Combine(tempDir.FullName, Path.GetFileName(p.ItemSpec)); + ZipFile.ExtractToDirectory(p.ItemSpec, packageTempPath, true); + + foreach (string f in Directory.EnumerateFiles(packageTempPath, "*", SearchOption.AllDirectories)) + { + // remove signatures so we don't later fail validation + if (Path.GetFileName(f) == ".signature.p7s") + { + File.Delete(f); + continue; + } + + var catalogFileEntry = new CatalogFileEntry(); + packageEntry.Files.Add(catalogFileEntry); + catalogFileEntry.Path = Utility.MakeRelativePath(f, packageTempPath); + AssemblyDefinition asm = null; + + // There seem to be some weird issues with using a file stream both for hashing and + // assembly loading, even closing it in between. Use a MemoryStream to avoid issues. + var memStream = new MemoryStream(); + using (var stream = File.OpenRead(f)) + { + stream.CopyTo(memStream); + } + + // First get the original hash of the file + memStream.Seek(0, SeekOrigin.Begin); + catalogFileEntry.OriginalHash = sha.ComputeHash(memStream); + + // Now try to read it as an assembly + memStream.Seek(0, SeekOrigin.Begin); + try + { + asm = AssemblyDefinition.ReadAssembly(memStream, new ReaderParameters(ReadingMode.Deferred)); + } + catch + { + // this is okay, it's not an assembly we can read + } + + // if we read it, now poison and write it back out + if (asm != null) + { + Poison(asm); + + try + { + // Cecil doesn't try to do some modifications until it writes out the file, + // and then throws after we've already truncated the file if it finds out it can't do them. + // Write to a memory stream first and then copy to the real stream if it suceeds. If it + // fails, we won't truncate the file and we will depend on hashes instead in that case. + using (var testMemStream = new MemoryStream()) + { + asm.Write(testMemStream); + testMemStream.Seek(0, SeekOrigin.Begin); + using (var stream = File.Open(f, FileMode.Create, FileAccess.ReadWrite)) + { + testMemStream.CopyTo(stream); + } + } + + // then get the hash of the now-poisoned file + using (var stream = File.OpenRead(f)) + { + catalogFileEntry.PoisonedHash = sha.ComputeHash(stream); + } + } + catch + { + // see above note in the try - this is okay. + } + } + } + + if (!string.IsNullOrWhiteSpace(MarkerFileName)) + { + var markerFilePath = Path.Combine(packageTempPath, MarkerFileName); + + if (File.Exists(markerFilePath)) + { + throw new ArgumentException($"Marker file name '{MarkerFileName}' is not sufficiently unique! Exists in '{p.ItemSpec}'.", nameof(MarkerFileName)); + } + + // mostly we just need to write something unique to this so it's not hashed as a matching file when we check it later. + // but it's also convenient to have the package catalog handy. + File.WriteAllText(markerFilePath, packageEntry.ToXml().ToString()); + } + + // create a temp file for this so if something goes wrong in the process we're not in too weird of a state + var poisonedPackageName = Path.GetFileName(p.ItemSpec) + ".poisoned"; + var poisonedPackagePath = Path.Combine(tempDir.FullName, poisonedPackageName); + ZipFile.CreateFromDirectory(packageTempPath, poisonedPackagePath); + + // Get the hash of the poisoned package (with poisoned marker file and poisoned assemblies inside) + using (var stream = File.OpenRead(poisonedPackagePath)) + { + packageEntry.PoisonedHash = sha.ComputeHash(stream); + } + File.Delete(p.ItemSpec); + File.Move(poisonedPackagePath, p.ItemSpec); + } + } + + // if we should write out the catalog, do that + if (!string.IsNullOrWhiteSpace(CatalogOutputFilePath)) + { + var outputFileDir = Path.GetDirectoryName(CatalogOutputFilePath); + if (!Directory.Exists(outputFileDir)) + { + Directory.CreateDirectory(outputFileDir); + } + File.WriteAllText(CatalogOutputFilePath, (new XElement("HashCatalog", + packageEntries.Select(p => p.ToXml()))).ToString()); + } + + tempDir.Delete(true); + return !Log.HasLoggedErrors; + } + + private void Poison(AssemblyDefinition asm) + { + foreach (var attr in asm.CustomAttributes) + { + if (this.AssemblyPropertiesToReplace.Any(p => p.Name == attr.AttributeType.Name)) + { + attr.ConstructorArguments.Clear(); + attr.ConstructorArguments.Add(new CustomAttributeArgument(asm.MainModule.TypeSystem.String, "POISONED by DotNetSourceBuild - Should not ship")); + } + } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj new file mode 100644 index 000000000..ab8eb402b --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj @@ -0,0 +1,35 @@ + + + + netcoreapp5.0 + true + $(LeakDetectionTasksBinDir) + + + + + 15.7.179 + + + 15.7.179 + + + Configuration=netstandard_Debug + Configuration=netstandard_Release + {D68133BD-1E63-496E-9EDE-4FBDBF77B486} + Mono.Cecil + + + Configuration=netstandard_Debug + Configuration=netstandard_Release + {D68133BD-1E63-496E-9EDE-4FBDBF77B486} + Mono.Cecil + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/NuGet.Config b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/NuGet.Config new file mode 100644 index 000000000..14757d86b --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/NuGet.Config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonMatch.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonMatch.cs new file mode 100644 index 000000000..0191b3a26 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonMatch.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + internal class PoisonMatch + { + const string ElementName = "Match"; + + internal string Package { get; set; } + internal string File { get; set; } + internal string PackageId { get; set; } + internal string PackageVersion { get; set; } + + public XElement ToXml() => new XElement(ElementName, + Package == null ? null : new XAttribute(nameof(Package), Package), + PackageId == null ? null : new XAttribute(nameof(PackageId), PackageId), + PackageVersion == null ? null : new XAttribute(nameof(PackageVersion), PackageVersion), + File == null ? null: new XAttribute(nameof(File), File) + ); + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonType.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonType.cs new file mode 100644 index 000000000..de5c35961 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonType.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + [Flags] + internal enum PoisonType + { + None = 0, + Hash = 1, + AssemblyAttribute = 2, + NupkgFile = 4, + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonedFileEntry.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonedFileEntry.cs new file mode 100644 index 000000000..1d98fd8cb --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonedFileEntry.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + internal class PoisonedFileEntry + { + const string ElementName = "File"; + + internal byte[] Hash { get; set; } + internal string Path { get; set; } + internal PoisonType Type { get; set; } + internal List Matches { get; } + + internal PoisonedFileEntry() + { + this.Matches = new List(); + } + + public XElement ToXml() => this.ToXml(ElementName); + + protected XElement ToXml(string myElementName) => new XElement(myElementName, + new XAttribute(nameof(Path), Path), + new XElement(nameof(Hash), Hash.ToHexString()), + new XElement(nameof(Type), Type.ToString()), + Matches.Select(m => m.ToXml()) + ); + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Utility.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Utility.cs new file mode 100644 index 000000000..48df5d261 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Utility.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + internal static class Utility + { + internal static string ToHexString(this byte[] bytes) + { + var sb = new StringBuilder(); + foreach (var b in bytes) + { + sb.Append(b.ToString("x2")); + } + return sb.ToString(); + } + + internal static byte[] ToBytes(this string hex) + { + var bytes = new List(); + for (var i = 0; i < hex.Length; i += 2) + { + bytes.Add(Convert.ToByte(hex.Substring(i, 2), 16)); + } + return bytes.ToArray(); + } + + internal static string MakeRelativePath(string filePath, string relativeTo) + { + // Uri.MakeRelativeUri requires the last slash + if (!relativeTo.EndsWith("/") && !relativeTo.EndsWith("\\")) + { + relativeTo += Path.DirectorySeparatorChar; + } + + var uri = new Uri(filePath); + var relativeToUri = new Uri(relativeTo); + return relativeToUri.MakeRelativeUri(uri).ToString(); + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/_._ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/_._ new file mode 100644 index 000000000..e69de29bb diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AddRidToRuntimeJson.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AddRidToRuntimeJson.cs new file mode 100644 index 000000000..5842d0d82 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AddRidToRuntimeJson.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Text; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System.Collections.Generic; +using System.Linq; +using NuGet.Versioning; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class AddRidToRuntimeJson:Task + { + /// + /// [OS name].[version]-[architecture] + /// + [Required] + public string Rid { get; set; } + + [Required] + public string RuntimeJson { get; set; } + + private string runtimesIdentifier = "runtimes"; + + public override bool Execute() + { + string[] ridParts = Rid.Split('-'); + string osNameAndVersion = ridParts[0]; + string[] osParts = osNameAndVersion.Split(new char[] { '.' }, 2); + + if (ridParts.Length < 1 || osParts.Length < 2) + { + throw new System.InvalidOperationException($"Unknown rid format {Rid}."); + } + + // Acquire Rid parts: + // osName + // version + // arch + string arch = ridParts[1]; + string osName = osParts[0]; + string version = osParts[1]; + + JObject projectRoot = ReadProject(RuntimeJson); + + if (projectRoot.SelectToken($"{runtimesIdentifier}.{osName}") == null) + { + AddRidToRuntimeGraph(projectRoot, osName, "linux"); + AddRidToRuntimeGraph(projectRoot, $"{osName}-{arch}", osName, $"linux-{arch}"); + } + if(projectRoot.SelectToken($"{runtimesIdentifier}.{osName}.{version}") == null) + { + AddRidToRuntimeGraph(projectRoot, $"{osName}.{version}", osName); + AddRidToRuntimeGraph(projectRoot, $"{osName}.{version}-{arch}", $"{osName}.{version}", $"{osName}-{arch}"); + } + + WriteProject(projectRoot, RuntimeJson); + return true; + } + + private void AddRidToRuntimeGraph(JObject projectRoot, string name, params string[] imports) + { + projectRoot[runtimesIdentifier][name] = new JObject(new JProperty("#import", new JArray(imports))); + } + + private static JObject ReadProject(string projectJsonPath) + { + using (TextReader projectFileReader = File.OpenText(projectJsonPath)) + { + var projectJsonReader = new JsonTextReader(projectFileReader); + var serializer = new JsonSerializer(); + return serializer.Deserialize(projectJsonReader); + } + } + private static void WriteProject(JObject projectRoot, string projectJsonPath) + { + string projectJson = JsonConvert.SerializeObject(projectRoot, Formatting.Indented) + Environment.NewLine; + + if (!File.Exists(projectJsonPath) || !projectJson.Equals(File.ReadAllText(projectJsonPath))) + { + Directory.CreateDirectory(Path.GetDirectoryName(projectJsonPath)); + File.WriteAllText(projectJsonPath, projectJson); + } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AddSourceToNuGetConfig.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AddSourceToNuGetConfig.cs new file mode 100755 index 000000000..a00a8a550 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AddSourceToNuGetConfig.cs @@ -0,0 +1,63 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + + +namespace Microsoft.DotNet.Build.Tasks +{ + /* + * This task adds a source to a well-formed NuGet.Config file. If a source with `SourceName` is already present, then + * the path of the source is changed. Otherwise, the source is added as the first source in the list, after any clear + * elements (if present). + */ + public class AddSourceToNuGetConfig : Task + { + [Required] + public string NuGetConfigFile { get; set; } + + [Required] + public string SourceName { get; set; } + + [Required] + public string SourcePath { get; set; } + + public override bool Execute() + { + XDocument d = XDocument.Load(NuGetConfigFile); + XElement packageSourcesElement = d.Root.Descendants().First(e => e.Name == "packageSources"); + XElement toAdd = new XElement("add", new XAttribute("key", SourceName), new XAttribute("value", SourcePath)); + XElement clearTag = new XElement("clear"); + + XElement exisitingSourceBuildElement = packageSourcesElement.Descendants().FirstOrDefault(e => e.Name == "add" && e.Attribute(XName.Get("key")).Value == SourceName); + XElement lastClearElement = packageSourcesElement.Descendants().LastOrDefault(e => e.Name == "clear"); + + if (exisitingSourceBuildElement != null) + { + exisitingSourceBuildElement.ReplaceWith(toAdd); + } + else if (lastClearElement != null) + { + lastClearElement.AddAfterSelf(toAdd); + } + else + { + packageSourcesElement.AddFirst(toAdd); + packageSourcesElement.AddFirst(clearTag); + } + + using (FileStream fs = new FileStream(NuGetConfigFile, FileMode.Create, FileAccess.ReadWrite)) + { + d.Save(fs); + } + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AzureConnectionStringBuildTask.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AzureConnectionStringBuildTask.cs new file mode 100644 index 000000000..b717d3450 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AzureConnectionStringBuildTask.cs @@ -0,0 +1,61 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Utilities; +using System.Text.RegularExpressions; + +namespace Microsoft.DotNet.Build.Tasks +{ + public abstract class AzureConnectionStringBuildTask : Task + { + /// + /// Azure Storage account connection string. Supersedes Account Key / Name. + /// Will cause errors if both are set. + /// + public string ConnectionString { get; set; } + + /// + /// The Azure account key used when creating the connection string. + /// When we fully deprecate these, can just make them get; only. + /// + public string AccountKey { get; set; } + + /// + /// The Azure account name used when creating the connection string. + /// When we fully deprecate these, can just make them get; only. + /// + public string AccountName { get; set; } + + public void ParseConnectionString() + { + if (!string.IsNullOrEmpty(ConnectionString)) + { + if (!(string.IsNullOrEmpty(AccountKey) && string.IsNullOrEmpty(AccountName))) + { + Log.LogError("If the ConnectionString property is set, you must not provide AccountKey / AccountName. These values will be deprecated in the future."); + } + else + { + Regex storageConnectionStringRegex = new Regex("AccountName=(?.+?);AccountKey=(?.+?);"); + + MatchCollection matches = storageConnectionStringRegex.Matches(ConnectionString); + if (matches.Count > 0) + { + // When we deprecate this format, we'll want to demote these to private + AccountName = matches[0].Groups["name"].Value; + AccountKey = matches[0].Groups["key"].Value; + } + else + { + Log.LogError("Error parsing connection string. Please review its value."); + } + } + } + else if (string.IsNullOrEmpty(AccountKey) || string.IsNullOrEmpty(AccountName)) + { + Log.LogError("Error, must provide either ConnectionString or AccountName with AccountKey"); + } + } + } +} \ No newline at end of file diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AzureHelper.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AzureHelper.cs new file mode 100644 index 000000000..ac08cfb3a --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AzureHelper.cs @@ -0,0 +1,461 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.Build.Tasks +{ + public static class AzureHelper + { + /// + /// The storage api version. + /// + public static readonly string StorageApiVersion = "2015-04-05"; + public const string DateHeaderString = "x-ms-date"; + public const string VersionHeaderString = "x-ms-version"; + public const string AuthorizationHeaderString = "Authorization"; + public const string CacheControlString = "x-ms-blob-cache-control"; + public const string ContentTypeString = "x-ms-blob-content-type"; + + public enum SasAccessType + { + Read, + Write, + }; + + public static string AuthorizationHeader( + string storageAccount, + string storageKey, + string method, + DateTime now, + HttpRequestMessage request, + string ifMatch = "", + string contentMD5 = "", + string size = "", + string contentType = "") + { + string stringToSign = string.Format( + "{0}\n\n\n{1}\n{5}\n{6}\n\n\n{2}\n\n\n\n{3}{4}", + method, + (size == string.Empty) ? string.Empty : size, + ifMatch, + GetCanonicalizedHeaders(request), + GetCanonicalizedResource(request.RequestUri, storageAccount), + contentMD5, + contentType); + byte[] signatureBytes = Encoding.UTF8.GetBytes(stringToSign); + string authorizationHeader; + using (HMACSHA256 hmacsha256 = new HMACSHA256(Convert.FromBase64String(storageKey))) + { + authorizationHeader = "SharedKey " + storageAccount + ":" + + Convert.ToBase64String(hmacsha256.ComputeHash(signatureBytes)); + } + + return authorizationHeader; + } + + public static string CreateContainerSasToken( + string accountName, + string containerName, + string key, + SasAccessType accessType, + int validityTimeInDays) + { + string signedPermissions = string.Empty; + switch (accessType) + { + case SasAccessType.Read: + signedPermissions = "r"; + break; + case SasAccessType.Write: + signedPermissions = "wdl"; + break; + default: + throw new ArgumentOutOfRangeException(nameof(accessType), accessType, "Unrecognized value"); + } + + string signedStart = DateTime.UtcNow.ToString("O"); + string signedExpiry = DateTime.UtcNow.AddDays(validityTimeInDays).ToString("O"); + string canonicalizedResource = "/blob/" + accountName + "/" + containerName; + string signedIdentifier = string.Empty; + string signedVersion = StorageApiVersion; + + string stringToSign = ConstructServiceStringToSign( + signedPermissions, + signedVersion, + signedExpiry, + canonicalizedResource, + signedIdentifier, + signedStart); + + byte[] signatureBytes = Encoding.UTF8.GetBytes(stringToSign); + string signature; + using (HMACSHA256 hmacSha256 = new HMACSHA256(Convert.FromBase64String(key))) + { + signature = Convert.ToBase64String(hmacSha256.ComputeHash(signatureBytes)); + } + + string sasToken = string.Format( + "?sv={0}&sr={1}&sig={2}&st={3}&se={4}&sp={5}", + WebUtility.UrlEncode(signedVersion), + WebUtility.UrlEncode("c"), + WebUtility.UrlEncode(signature), + WebUtility.UrlEncode(signedStart), + WebUtility.UrlEncode(signedExpiry), + WebUtility.UrlEncode(signedPermissions)); + + return sasToken; + } + + public static string GetCanonicalizedHeaders(HttpRequestMessage request) + { + StringBuilder sb = new StringBuilder(); + List headerNameList = (from headerName in request.Headers + where + headerName.Key.ToLowerInvariant() + .StartsWith("x-ms-", StringComparison.Ordinal) + select headerName.Key.ToLowerInvariant()).ToList(); + headerNameList.Sort(); + foreach (string headerName in headerNameList) + { + StringBuilder builder = new StringBuilder(headerName); + string separator = ":"; + foreach (string headerValue in GetHeaderValues(request.Headers, headerName)) + { + string trimmedValue = headerValue.Replace("\r\n", string.Empty); + builder.Append(separator); + builder.Append(trimmedValue); + separator = ","; + } + + sb.Append(builder); + sb.Append("\n"); + } + + return sb.ToString(); + } + + public static string GetCanonicalizedResource(Uri address, string accountName) + { + StringBuilder str = new StringBuilder(); + StringBuilder builder = new StringBuilder("/"); + builder.Append(accountName); + builder.Append(address.AbsolutePath); + str.Append(builder); + Dictionary> queryKeyValues = ExtractQueryKeyValues(address); + Dictionary> dictionary = GetCommaSeparatedList(queryKeyValues); + + foreach (KeyValuePair> pair in dictionary.OrderBy(p => p.Key)) + { + StringBuilder stringBuilder = new StringBuilder(string.Empty); + stringBuilder.Append(pair.Key + ":"); + string commaList = string.Join(",", pair.Value); + stringBuilder.Append(commaList); + str.Append("\n"); + str.Append(stringBuilder); + } + + return str.ToString(); + } + + public static List GetHeaderValues(HttpRequestHeaders headers, string headerName) + { + List list = new List(); + IEnumerable values; + headers.TryGetValues(headerName, out values); + if (values != null) + { + list.Add((values.FirstOrDefault() ?? string.Empty).TrimStart(null)); + } + + return list; + } + + private static bool IsWithinRetryRange(HttpStatusCode statusCode) + { + // Retry on http client and server error codes (4xx - 5xx) as well as redirect + + var rawStatus = (int)statusCode; + if (rawStatus == 302) + return true; + else if (rawStatus >= 400 && rawStatus <= 599) + return true; + else + return false; + } + + public static async Task RequestWithRetry(TaskLoggingHelper loggingHelper, HttpClient client, + Func createRequest, Func validationCallback = null, int retryCount = 5, + int retryDelaySeconds = 5) + { + if (loggingHelper == null) + throw new ArgumentNullException(nameof(loggingHelper)); + if (client == null) + throw new ArgumentNullException(nameof(client)); + if (createRequest == null) + throw new ArgumentNullException(nameof(createRequest)); + if (retryCount < 1) + throw new ArgumentException(nameof(retryCount)); + if (retryDelaySeconds < 1) + throw new ArgumentException(nameof(retryDelaySeconds)); + + int retries = 0; + HttpResponseMessage response = null; + + // add a bit of randomness to the retry delay + var rng = new Random(); + + while (retries < retryCount) + { + if (retries > 0) + { + if (response != null) + { + response.Dispose(); + response = null; + } + + int delay = retryDelaySeconds * retries * rng.Next(1, 5); + loggingHelper.LogMessage(MessageImportance.Low, "Waiting {0} seconds before retry", delay); + await System.Threading.Tasks.Task.Delay(delay * 1000); + } + + try + { + using (var request = createRequest()) + response = await client.SendAsync(request); + } + catch (Exception e) + { + loggingHelper.LogWarningFromException(e, true); + + // if this is the final iteration let the exception bubble up + if (retries + 1 == retryCount) + throw; + } + + // response can be null if we fail to send the request + if (response != null) + { + if (validationCallback == null) + { + // check if the response code is within the range of failures + if (!IsWithinRetryRange(response.StatusCode)) + { + return response; + } + } + else + { + bool isSuccess = validationCallback(response); + if (!isSuccess) + { + loggingHelper.LogMessage("Validation callback returned retry for status code {0}", response.StatusCode); + } + else + { + loggingHelper.LogMessage("Validation callback returned success for status code {0}", response.StatusCode); + return response; + } + } + } + + ++retries; + } + + // retry count exceeded + loggingHelper.LogWarning("Retry count {0} exceeded", retryCount); + + // set some default values in case response is null + var statusCode = "None"; + var contentStr = "Null"; + if (response != null) + { + statusCode = response.StatusCode.ToString(); + contentStr = await response.Content.ReadAsStringAsync(); + response.Dispose(); + } + + throw new HttpRequestException($"Request {createRequest().RequestUri} failed with status {statusCode}. Response : {contentStr}"); + } + + private static string ConstructServiceStringToSign( + string signedPermissions, + string signedVersion, + string signedExpiry, + string canonicalizedResource, + string signedIdentifier, + string signedStart, + string signedIP = "", + string signedProtocol = "", + string rscc = "", + string rscd = "", + string rsce = "", + string rscl = "", + string rsct = "") + { + // constructing string to sign based on spec in https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx + var stringToSign = string.Join( + "\n", + signedPermissions, + signedStart, + signedExpiry, + canonicalizedResource, + signedIdentifier, + signedIP, + signedProtocol, + signedVersion, + rscc, + rscd, + rsce, + rscl, + rsct); + return stringToSign; + } + + private static Dictionary> ExtractQueryKeyValues(Uri address) + { + Dictionary> values = new Dictionary>(); + //Decode this to allow the regex to pull out the correct groups for signing + address = new Uri(WebUtility.UrlDecode(address.ToString())); + Regex newreg = new Regex(@"(?:\?|&)([^=]+)=([^&]+)"); + MatchCollection matches = newreg.Matches(address.Query); + foreach (Match match in matches) + { + string key, value; + if (!string.IsNullOrEmpty(match.Groups[1].Value)) + { + key = match.Groups[1].Value; + value = match.Groups[2].Value; + } + else + { + key = match.Groups[3].Value; + value = match.Groups[4].Value; + } + + HashSet setOfValues; + if (values.TryGetValue(key, out setOfValues)) + { + setOfValues.Add(value); + } + else + { + HashSet newSet = new HashSet { value }; + values.Add(key, newSet); + } + } + + return values; + } + + private static Dictionary> GetCommaSeparatedList( + Dictionary> queryKeyValues) + { + Dictionary> dictionary = new Dictionary>(); + + foreach (string queryKeys in queryKeyValues.Keys) + { + HashSet setOfValues; + queryKeyValues.TryGetValue(queryKeys, out setOfValues); + List list = new List(); + list.AddRange(setOfValues); + list.Sort(); + string commaSeparatedValues = string.Join(",", list); + string key = queryKeys.ToLowerInvariant(); + HashSet setOfValues2; + if (dictionary.TryGetValue(key, out setOfValues2)) + { + setOfValues2.Add(commaSeparatedValues); + } + else + { + HashSet newSet = new HashSet { commaSeparatedValues }; + dictionary.Add(key, newSet); + } + } + + return dictionary; + } + + public static Func RequestMessage(string method, string url, string accountName, string accountKey, List> additionalHeaders = null, string body = null) + { + Func requestFunc = () => + { + HttpMethod httpMethod = HttpMethod.Get; + if (method == "PUT") + { + httpMethod = HttpMethod.Put; + } + else if (method == "DELETE") + { + httpMethod = HttpMethod.Delete; + } + DateTime dateTime = DateTime.UtcNow; + var request = new HttpRequestMessage(httpMethod, url); + request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture)); + request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); + if (additionalHeaders != null) + { + foreach (Tuple additionalHeader in additionalHeaders) + { + request.Headers.Add(additionalHeader.Item1, additionalHeader.Item2); + } + } + if (body != null) + { + request.Content = new StringContent(body); + request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( + accountName, + accountKey, + method, + dateTime, + request, + "", + "", + request.Content.Headers.ContentLength.ToString(), + request.Content.Headers.ContentType.ToString())); + } + else + { + request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( + accountName, + accountKey, + method, + dateTime, + request)); + } + return request; + }; + return requestFunc; + } + + public static string GetRootRestUrl(string accountName) + { + return $"https://{accountName}.blob.core.windows.net"; + } + + public static string GetContainerRestUrl(string accountName, string containerName) + { + return $"{GetRootRestUrl(accountName)}/{containerName}"; + } + + public static string GetBlobRestUrl(string accountName, string containerName, string blob) + { + return $"{GetContainerRestUrl(accountName, containerName)}/{blob}"; + } + } +} \ No newline at end of file diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/BuildTask.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/BuildTask.cs new file mode 100644 index 000000000..f4fd687e9 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/BuildTask.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + public abstract partial class BuildTask : ITask + { + private TaskLoggingHelper _log = null; + + internal TaskLoggingHelper Log + { + get { return _log ?? (_log = new TaskLoggingHelper(this)); } + } + + public BuildTask() + { + } + + public IBuildEngine BuildEngine + { + get; + set; + } + + public ITaskHost HostObject + { + get; + set; + } + + public abstract bool Execute(); + } +} \ No newline at end of file diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/DownloadFileSB.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/DownloadFileSB.cs new file mode 100644 index 000000000..680327baf --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/DownloadFileSB.cs @@ -0,0 +1,409 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +// This task is sourced from https://github.com/microsoft/msbuild/blob/04e508c36f9c1fe826264aef7c26ffb8f16e9bdc/src/Tasks/DownloadFile.cs +// Contains further modifications in followup commits. +// It alleviates the problem of time outs on DownloadFile Task. We are not the version of msbuild that has this fix, hence we have to locally +// build it to get rid of the issue. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Microsoft.DotNet.Build.Tasks; +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.Build.Tasks +{ + /// + /// Represents a task that can download a file. + /// + public sealed class DownloadFileSB : BuildTask, ICancelableTask + { + private readonly CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource(); + + /// + /// Gets or sets an optional filename for the destination file. By default, the filename is derived from the if possible. + /// + public ITaskItem DestinationFileName { get; set; } + + /// + /// Gets or sets a that specifies the destination folder to download the file to. + /// + [Required] + public ITaskItem DestinationFolder { get; set; } + + /// + /// Gets or sets a that contains details about the downloaded file. + /// + [Output] + public ITaskItem DownloadedFile { get; set; } + + /// + /// Gets or sets an optional number of times to retry if possible. + /// + public int Retries { get; set; } + + /// + /// Gets or sets the number of milliseconds to wait before retrying. + /// + public int RetryDelayMilliseconds { get; set; } = 5 * 1000; + + /// + /// Gets or sets an optional value indicating whether or not the download should be skipped if the file is up-to-date. + /// + public bool SkipUnchangedFiles { get; set; } = true; + + /// + /// Gets or sets the URL to download. + /// + [Required] + public string SourceUrl { get; set; } + + /// + /// Gets or sets the timeout for a successful download. If exceeded, the download continues + /// for another two timeout durations before failing. This makes it sometimes possible to + /// determine whether the timeout is just a little too short, or if the download would never + /// have finished. + /// + public string TimeoutSeconds { get; set; } + + /// + /// Gets or sets a to use. This is used by unit tests to mock a connection to a remote server. + /// + internal HttpMessageHandler HttpMessageHandler { get; set; } + + /// + public void Cancel() + { + _cancellationTokenSource.Cancel(); + } + + public override bool Execute() + { + return ExecuteAsync().GetAwaiter().GetResult(); + } + + private async Task ExecuteAsync() + { + if (!Uri.TryCreate(SourceUrl, UriKind.Absolute, out Uri uri)) + { + Log.LogError($"DownloadFileSB.ErrorInvalidUrl {SourceUrl}"); + return false; + } + + int retryAttemptCount = 0; + + CancellationToken cancellationToken = _cancellationTokenSource.Token; + + var startTime = DateTime.UtcNow; + + // Use the same API for the "success timeout" and the "would it ever succeed" timeout. + var timeout = TimeSpan.Zero; + var successCancellationTokenSource = new CancellationTokenSource(); + + if (double.TryParse(TimeoutSeconds, out double timeoutSeconds)) + { + timeout = TimeSpan.FromSeconds(timeoutSeconds); + Log.LogMessage(MessageImportance.High, $"DownloadFileSB timeout set to {timeout}"); + + successCancellationTokenSource.CancelAfter(timeout); + _cancellationTokenSource.CancelAfter((int)(timeout.TotalMilliseconds * 3)); + } + + while (true) + { + try + { + await DownloadAsync(uri, cancellationToken); + break; + } + catch (OperationCanceledException e) when (e.CancellationToken == cancellationToken) + { + // This task is being cancelled. Exit the loop. + break; + } + catch (Exception e) + { + bool canRetry = IsRetriable(e, out Exception actualException) && retryAttemptCount++ < Retries; + + if (canRetry) + { + Log.LogWarning($"DownloadFileSB.Retrying {SourceUrl} {retryAttemptCount + 1} {RetryDelayMilliseconds} {actualException}"); + + try + { + await Task.Delay(RetryDelayMilliseconds, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException delayException) when (delayException.CancellationToken == cancellationToken) + { + // This task is being cancelled, exit the loop + break; + } + } + else + { + Log.LogError($"DownloadFileSB.ErrorDownloading {SourceUrl} {actualException}"); + break; + } + } + } + + var finishTime = DateTime.UtcNow; + + if (successCancellationTokenSource.IsCancellationRequested) + { + string error = $"{TimeoutSeconds} second timeout exceeded"; + + if (!_cancellationTokenSource.IsCancellationRequested) + { + error += + $", but download completed after {finishTime - startTime}. " + + $"Try increasing timeout from {TimeoutSeconds} if this is acceptable."; + } + else + { + error += + $", and didn't complete within leeway after {finishTime - startTime}. " + + $"The download was likely never going to terminate. Investigate logs and " + + $"add additional logging if necessary."; + } + + Log.LogError(error); + } + else + { + Log.LogMessage( + MessageImportance.High, + $"DownloadFileSB.Downloading Complete! Elapsed: {finishTime - startTime}"); + } + + return !_cancellationTokenSource.IsCancellationRequested && !Log.HasLoggedErrors; + } + + /// + /// Attempts to download the file. + /// + /// The parsed of the request. + private async Task DownloadAsync(Uri uri, CancellationToken cancellationToken) + { + // The main reason to use HttpClient vs WebClient is because we can pass a message handler for unit tests to mock + using (var client = new HttpClient(HttpMessageHandler ?? new HttpClientHandler(), disposeHandler: true)) + { + // Only get the response without downloading the file so we can determine if the file is already up-to-date + using (HttpResponseMessage response = await client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false)) + { + try + { + response.EnsureSuccessStatusCode(); + } + catch (HttpRequestException e) + { + // HttpRequestException does not have the status code so its wrapped and thrown here so that later on we can determine + // if a retry is possible based on the status code + throw new CustomHttpRequestException(e.Message, e.InnerException, response.StatusCode); + } + + if (!TryGetFileName(response, out string filename)) + { + Log.LogError($"DownloadFileSB.ErrorUnknownFileName {SourceUrl} {nameof(DestinationFileName)}"); + return; + } + + DirectoryInfo destinationDirectory = Directory.CreateDirectory(DestinationFolder.ItemSpec); + + var destinationFile = new FileInfo(Path.Combine(destinationDirectory.FullName, filename)); + + // The file is considered up-to-date if its the same length. This could be inaccurate, we can consider alternatives in the future + if (ShouldSkip(response, destinationFile)) + { + Log.LogMessage(MessageImportance.Normal, $"DownloadFileSB.DidNotDownloadBecauseOfFileMatch {SourceUrl}", destinationFile.FullName, nameof(SkipUnchangedFiles), "true"); + + DownloadedFile = new TaskItem(destinationFile.FullName); + + return; + } + + var progressMonitorCancellationTokenSource = new CancellationTokenSource(); + CancellationToken progressMonitorToken = progressMonitorCancellationTokenSource.Token; + + try + { + cancellationToken.ThrowIfCancellationRequested(); + + var startTime = DateTime.UtcNow; + + var progressMonitor = Task.Run( + async () => + { + while (!progressMonitorToken.IsCancellationRequested) + { + destinationFile.Refresh(); + if (destinationFile.Exists) + { + long current = destinationFile.Length; + long total = response.Content.Headers.ContentLength ?? 1; + var elapsed = DateTime.UtcNow - startTime; + double kbytesPerSecond = current / elapsed.TotalSeconds / 1000.0; + + Log.LogMessage( + MessageImportance.High, + $"Progress... {elapsed}, " + + $"current file size {current / (double)total:00.0%} " + + $"({destinationFile.Length:#,0} / {total:#,0}) " + + $"~ {kbytesPerSecond:#,0.00} kB/s"); + } + await Task.Delay(TimeSpan.FromSeconds(5), progressMonitorToken); + } + }, + progressMonitorToken) + .ConfigureAwait(false); + + using (var target = new FileStream(destinationFile.FullName, FileMode.Create, FileAccess.Write, FileShare.None)) + { + Log.LogMessage( + MessageImportance.High, + $"DownloadFileSB.Downloading {SourceUrl} to " + + $"{destinationFile.FullName}"); + + Log.LogMessage( MessageImportance.Low, $"All response headers:\n{response.Headers}"); + Log.LogMessage( MessageImportance.Low, $"All content headers:\n{response.Content.Headers}"); + + using (Stream responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) + { + await responseStream.CopyToAsync(target, 1024, cancellationToken).ConfigureAwait(false); + } + + Log.LogMessage(MessageImportance.High, $"DownloadFileSB.StreamCopyComplete {SourceUrl}"); + + DownloadedFile = new TaskItem(destinationFile.FullName); + } + } + finally + { + if (DownloadedFile == null) + { + // Delete the file if anything goes wrong during download. This could be destructive but we don't want to leave + // partially downloaded files on disk either. Alternatively we could download to a temporary location and copy + // on success but we are concerned about the added I/O + destinationFile.Delete(); + } + + progressMonitorCancellationTokenSource.Cancel(); + } + } + } + } + + /// + /// Determines if the specified exception is considered retriable. + /// + /// The originally thrown exception. + /// The actual exception to be used for logging errors. + /// true if the exception is retriable, otherwise false. + private static bool IsRetriable(Exception exception, out Exception actualException) + { + actualException = exception; + + // Get aggregate inner exception + if (actualException is AggregateException aggregateException && aggregateException.InnerException != null) + { + actualException = aggregateException.InnerException; + } + + // Some HttpRequestException have an inner exception that has the real error + if (actualException is HttpRequestException httpRequestException && httpRequestException.InnerException != null) + { + actualException = httpRequestException.InnerException; + + // An IOException inside of a HttpRequestException means that something went wrong while downloading + if (actualException is IOException) + { + return true; + } + } + + if (actualException is CustomHttpRequestException customHttpRequestException) + { + // A wrapped CustomHttpRequestException has the status code from the error + switch (customHttpRequestException.StatusCode) + { + case HttpStatusCode.InternalServerError: + case HttpStatusCode.RequestTimeout: + return true; + } + } + + if (actualException is WebException webException) + { + // WebException is thrown when accessing the Content of the response + switch (webException.Status) + { + // Don't retry on anything that cannot be compensated for + case WebExceptionStatus.TrustFailure: + case WebExceptionStatus.MessageLengthLimitExceeded: + case WebExceptionStatus.RequestProhibitedByCachePolicy: + case WebExceptionStatus.RequestProhibitedByProxy: + return false; + + default: + // Retry on all other WebExceptions + return true; + } + } + + return false; + } + + /// + /// Attempts to get the file name to use when downloading the file. + /// + /// The with information about the response. + /// Receives the name of the file. + /// true if a file name could be determined, otherwise false. + private bool TryGetFileName(HttpResponseMessage response, out string filename) + { + if (response == null) + { + throw new ArgumentNullException(nameof(response)); + } + + // Not all URIs contain a file name so users will have to specify one + // Example: http://www.download.com/file/1/ + + filename = !String.IsNullOrWhiteSpace(DestinationFileName?.ItemSpec) + ? DestinationFileName.ItemSpec // Get the file name from what the user specified + : response.Content?.Headers?.ContentDisposition?.FileName // Attempt to get the file name from the content-disposition header value + ?? Path.GetFileName(response.RequestMessage.RequestUri.LocalPath); // Otherwise attempt to get a file name from the URI + + return !String.IsNullOrWhiteSpace(filename); + } + + /// + /// Represents a wrapper around the that also contains the . + /// + private sealed class CustomHttpRequestException : HttpRequestException + { + public CustomHttpRequestException(string message, Exception inner, HttpStatusCode statusCode) + : base(message, inner) + { + StatusCode = statusCode; + } + + public HttpStatusCode StatusCode { get; } + } + + private bool ShouldSkip(HttpResponseMessage response, FileInfo destinationFile) + { + return SkipUnchangedFiles + && destinationFile.Exists + && destinationFile.Length == response.Content.Headers.ContentLength + && response.Content.Headers.LastModified.HasValue + && destinationFile.LastWriteTimeUtc > response.Content.Headers.LastModified.Value.UtcDateTime; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/EnumerableExtensions.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/EnumerableExtensions.cs new file mode 100644 index 000000000..74102cff0 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/EnumerableExtensions.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks +{ + internal static class EnumerableExtensions + { + public static IEnumerable NullAsEmpty(this IEnumerable source) + { + return source ?? Enumerable.Empty(); + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/FixPathSeparator.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/FixPathSeparator.cs new file mode 100755 index 000000000..263db19cf --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/FixPathSeparator.cs @@ -0,0 +1,86 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + /* + * This task replaces both types of path separators ('/' and '\') with the separator for the current + * platform. This workaround a NuGet issue where `nuget pack` does not translate path separators causing + * packages that don't appear to have the right assets in them. + */ + public class FixPathSeparator : Task + { + [Required] + public ITaskItem[] NuSpecFiles { get; set; } + + public override bool Execute() + { + foreach (ITaskItem item in NuSpecFiles) + { + string pathToNuSpec = item.GetMetadata("FullPath"); + + XDocument doc = XDocument.Load(pathToNuSpec); + + XElement contentFilesElement = doc.ElementIgnoringNamespace("package").ElementIgnoringNamespace("metadata").ElementIgnoringNamespace("contentFiles"); + XElement filesElement = doc.ElementIgnoringNamespace("package").ElementIgnoringNamespace("files"); + + if (contentFilesElement != null) + { + foreach (XElement element in contentFilesElement.ElementsIgnroingNamespace("files")) + { + UpdateDirectorySeperatorInAttribute(element, "include"); + UpdateDirectorySeperatorInAttribute(element, "exclude"); + } + } + + if (filesElement != null) + { + foreach (XElement element in filesElement.ElementsIgnroingNamespace("file")) + { + UpdateDirectorySeperatorInAttribute(element, "src"); + UpdateDirectorySeperatorInAttribute(element, "target"); + UpdateDirectorySeperatorInAttribute(element, "exclude"); + } + } + + using (FileStream fs = File.Open(pathToNuSpec, FileMode.Truncate)) + { + doc.Save(fs); + } + } + + return true; + } + + private static void UpdateDirectorySeperatorInAttribute(XElement element, XName name) + { + XAttribute attribute = element.Attribute(name); + + if (attribute != null) + { + element.SetAttributeValue(name, attribute.Value.Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar)); + } + } + } + + static class XContainerExtensions + { + public static IEnumerable ElementsIgnroingNamespace(this XContainer container, XName elementName) + { + return container.Elements().Where(e => e.Name.LocalName == elementName.LocalName); + } + + public static XElement ElementIgnoringNamespace(this XContainer container, XName elementName) + { + return container.ElementsIgnroingNamespace(elementName).FirstOrDefault(); + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/GetSourceBuiltNupkgCacheConflicts.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/GetSourceBuiltNupkgCacheConflicts.cs new file mode 100644 index 000000000..7eb24ae1a --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/GetSourceBuiltNupkgCacheConflicts.cs @@ -0,0 +1,153 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Packaging.Core; +using NuGet.Versioning; +using System; +using System.IO; +using System.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks +{ + /// + /// For each source-built nupkg info given, ensure that if the package cache contains a package + /// with the same id and version, the cached nupkg is the same as the source-built one. + /// + /// If the package cache contains a package with the same package id and version as a + /// source-built one, nuget restore short-circuits and doesn't look for the source-built one. + /// This usually results in prebuilt packages being used, which can either break the build or + /// end up in the outputs. + /// + public class GetSourceBuiltNupkgCacheConflicts : Task + { + /// + /// Items containing package id and version of each source-built package. + /// ReadNuGetPackageInfos is recommended to generate these. + /// + /// %(Identity): Path to the original nupkg. + /// %(PackageId): Identity of the package. + /// %(PackageVersion): Version of the package. + /// + [Required] + public ITaskItem[] SourceBuiltPackageInfos { get; set; } + + /// + /// Package cache dir containing nupkgs to compare. Path is expected to be like: + /// + /// {PackageCacheDir}/{lowercase id}/{version}/{lowercase id}.{version}.nupkg + /// + [Required] + public string PackageCacheDir { get; set; } + + /// + /// Paths to packages to compare against when conflicts are detected. Knowing where the + /// package in the cache came from can help diagnose a conflict. For example, is it from + /// prebuilt/source-built? Or does the build not have the nupkg anywhere else, and + /// therefore it most likely came from the internet? + /// + public string[] KnownOriginPackagePaths { get; set; } + + [Output] + public ITaskItem[] ConflictingPackageInfos { get; set; } + + public override bool Execute() + { + DateTime startTime = DateTime.Now; + + var knownNupkgs = new Lazy>(() => + { + Log.LogMessage( + MessageImportance.Low, + $"Reading all {nameof(KnownOriginPackagePaths)} package identities to search " + + "for conflicting package origin..."); + + return KnownOriginPackagePaths.NullAsEmpty().ToLookup( + ReadNuGetPackageInfos.ReadIdentity, + path => path); + }); + + ConflictingPackageInfos = SourceBuiltPackageInfos + .Where(item => + { + string sourceBuiltPath = item.ItemSpec; + string id = item.GetMetadata("PackageId"); + string version = item.GetMetadata("PackageVersion"); + + string packageCachePath = Path.Combine( + PackageCacheDir, + id.ToLowerInvariant(), + version, + $"{id.ToLowerInvariant()}.{version}.nupkg"); + + if (!File.Exists(packageCachePath)) + { + Log.LogMessage( + MessageImportance.Low, + $"OK: Package not found in package cache: {id} {version}"); + return false; + } + + Log.LogMessage( + MessageImportance.Low, + $"Package id/version found in package cache, verifying: {id} {version}"); + + byte[] packageCacheBytes = File.ReadAllBytes(packageCachePath); + + if (packageCacheBytes.SequenceEqual(File.ReadAllBytes(sourceBuiltPath))) + { + Log.LogMessage( + MessageImportance.Low, + $"OK: Package in cache is identical to source-built: {id} {version}"); + return false; + } + + Log.LogMessage( + MessageImportance.Low, + "BAD: Source-built nupkg is not byte-for-byte identical " + + $"to nupkg in cache: {id} {version}"); + + var ident = new PackageIdentity(id, NuGetVersion.Parse(version)); + + string message = null; + + foreach (string knownNupkg in knownNupkgs.Value[ident]) + { + if (packageCacheBytes.SequenceEqual(File.ReadAllBytes(knownNupkg))) + { + Log.LogMessage( + MessageImportance.Low, + $"Found identity match with identical contents: {knownNupkg}"); + + message = (message ?? "Nupkg found at") + $" '{knownNupkg}'"; + } + else + { + Log.LogMessage( + MessageImportance.Low, + $"Package identity match, but contents differ: {knownNupkg}"); + } + } + + item.SetMetadata( + "WarningMessage", + message ?? + "Origin nupkg not found in build directory. It may have been " + + "downloaded by NuGet restore."); + + return true; + }) + .ToArray(); + + // Tell the user about this task, in case it takes a while. + Log.LogMessage( + MessageImportance.High, + "Checked cache for conflicts with source-built nupkgs. " + + $"Took {DateTime.Now - startTime}"); + + return !Log.HasLoggedErrors; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Log.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Log.cs new file mode 100644 index 000000000..83d587c0a --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Log.cs @@ -0,0 +1,118 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks.Packaging +{ + internal class Log : ILog + { + private readonly TaskLoggingHelper _logger; + public Log(TaskLoggingHelper logger) + { + _logger = logger; + } + + public void LogError(string message, params object[] messageArgs) + { + _logger.LogError(message, messageArgs); + } + + public void LogMessage(string message, params object[] messageArgs) + { + _logger.LogMessage(message, messageArgs); + } + + public void LogMessage(LogImportance importance, string message, params object[] messageArgs) + { + _logger.LogMessage((MessageImportance)importance, message, messageArgs); + } + + public void LogWarning(string message, params object[] messageArgs) + { + _logger.LogWarning(message, messageArgs); + } + + public bool HasLoggedErrors { get { return _logger.HasLoggedErrors; } } + } + + public enum LogImportance + { + Low = MessageImportance.Low, + Normal = MessageImportance.Normal, + High = MessageImportance.High + } + + + public interface ILog + { + // + // Summary: + // Logs an error with the specified message. + // + // Parameters: + // message: + // The message. + // + // messageArgs: + // Optional arguments for formatting the message string. + // + // Exceptions: + // T:System.ArgumentNullException: + // message is null. + void LogError(string message, params object[] messageArgs); + + // + // Summary: + // Logs a message with the specified string. + // + // Parameters: + // message: + // The message. + // + // messageArgs: + // The arguments for formatting the message. + // + // Exceptions: + // T:System.ArgumentNullException: + // message is null. + void LogMessage(string message, params object[] messageArgs); + + // + // Summary: + // Logs a message with the specified string and importance. + // + // Parameters: + // importance: + // One of the enumeration values that specifies the importance of the message. + // + // message: + // The message. + // + // messageArgs: + // The arguments for formatting the message. + // + // Exceptions: + // T:System.ArgumentNullException: + // message is null. + void LogMessage(LogImportance importance, string message, params object[] messageArgs); + + // + // Summary: + // Logs a warning with the specified message. + // + // Parameters: + // message: + // The message. + // + // messageArgs: + // Optional arguments for formatting the message string. + // + // Exceptions: + // T:System.ArgumentNullException: + // message is null. + void LogWarning(string message, params object[] messageArgs); + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Microsoft.DotNet.SourceBuild.Tasks.XPlat.csproj b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Microsoft.DotNet.SourceBuild.Tasks.XPlat.csproj new file mode 100755 index 000000000..08d19f382 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Microsoft.DotNet.SourceBuild.Tasks.XPlat.csproj @@ -0,0 +1,27 @@ + + + + netcoreapp5.0 + $(XPlatTasksBinDir) + + + + + 15.7.179 + + + 15.7.179 + + + 15.7.179 + + + 15.7.179 + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Models/VersionDetailsXml.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Models/VersionDetailsXml.cs new file mode 100644 index 000000000..89e78085e --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Models/VersionDetailsXml.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Xml.Serialization; + +namespace Microsoft.DotNet.SourceBuild.Tasks.Models +{ + [XmlRoot("Dependencies")] + public class VersionDetails + { + [XmlArray("ToolsetDependencies")] + public Dependency[] ToolsetDependencies { get; set; } + [XmlArray("ProductDependencies")] + public Dependency[] ProductDependencies { get; set; } + } + + public class Dependency + { + [XmlAttribute] + public string Name { get; set; } + [XmlAttribute] + public string Version { get; set; } + [XmlAttribute] + public string CoherentParentDependency { get; set; } + [XmlAttribute] + public bool Pinned { get; set; } + // Uri type isn't serializable, so use a string instead + public string Uri { get; set; } + public string Sha { get; set; } + [XmlElement("RepoName")] + public string[] RepoNames { get; set; } + + public override string ToString() + { + return $"{Name}@{Version} ({Uri}@{Sha})"; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/NuGetPack.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/NuGetPack.cs new file mode 100644 index 000000000..d463f53e8 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/NuGetPack.cs @@ -0,0 +1,376 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using NuGet; +using NuGet.Versioning; +using NuGet.Packaging; +using System; +using System.IO; +using System.Collections.Generic; +using System.Linq; +using NuGet.Common; + +namespace Microsoft.DotNet.Build.Tasks.Packaging +{ + public class NuGetPack : PackagingTask + { + /// + /// Target file paths to exclude when building the lib package for symbol server scenario + /// Copied from https://github.com/NuGet/NuGet.Client/blob/59433c7bacaae435a2cfe343cd441ea710579304/src/NuGet.Core/NuGet.Commands/PackCommandRunner.cs#L48 + /// + private static readonly string[] _libPackageExcludes = new[] { + @"**\*.pdb".Replace('\\', Path.DirectorySeparatorChar), + @"src\**\*".Replace('\\', Path.DirectorySeparatorChar) + }; + + /// + /// Target file paths to exclude when building the symbols package for symbol server scenario + /// + private static readonly string[] _symbolPackageExcludes = new[] { + @"content\**\*".Replace('\\', Path.DirectorySeparatorChar), + @"tools\**\*.ps1".Replace('\\', Path.DirectorySeparatorChar) + }; + + private static readonly string _defaultPackedPackagePrefix = "transport"; + private static readonly string _symbolsPackageExtension = ".symbols.nupkg"; + private static readonly string _packageExtension = ".nupkg"; + + [Required] + public ITaskItem[] Nuspecs + { + get; + set; + } + + [Required] + public string OutputDirectory + { + get; + set; + } + + public string BaseDirectory + { + get; + set; + } + + public string PackageVersion + { + get; + set; + } + + public bool ExcludeEmptyDirectories + { + get; + set; + } + // Create an additional ".symbols.nupkg" package + public bool CreateSymbolPackage + { + get; + set; + } + // Include symbols in standard package + public bool IncludeSymbolsInPackage + { + get; + set; + } + // Create an additional "packed package" that includes lib and src / symbols + public bool CreatePackedPackage + { + get; + set; + } + /// + /// Nuspec files can contain properties that are substituted with values at pack time + /// This task property passes through the nuspect properties. + /// Each item is a string with the syntax = + /// String validation for and is deffered to the Nuget APIs + /// + public ITaskItem[] NuspecProperties + { + get; + set; + } + + public ITaskItem[] AdditionalLibPackageExcludes + { + get; + set; + } + + public ITaskItem[] AdditionalSymbolPackageExcludes + { + get; + set; + } + + /// + /// If set, the symbol package is placed in the given directory. Otherwise OutputDirectory is used. + /// + public string SymbolPackageOutputDirectory + { + get; + set; + } + + public string PackedPackageNamePrefix + { + get; + set; + } + + public override bool Execute() + { + if (Nuspecs == null || Nuspecs.Length == 0) + { + Log.LogError("Nuspecs argument must be specified"); + return false; + } + + if (String.IsNullOrEmpty(OutputDirectory)) + { + Log.LogError("OuputDirectory argument must be specified"); + return false; + } + + if (!Directory.Exists(OutputDirectory)) + { + Directory.CreateDirectory(OutputDirectory); + } + + Func nuspecPropertyProvider = GetNuspecPropertyProviderFunction(NuspecProperties); + + foreach (var nuspec in Nuspecs) + { + string nuspecPath = nuspec.GetMetadata("FullPath"); + + if (!File.Exists(nuspecPath)) + { + Log.LogError($"Nuspec {nuspecPath} does not exist"); + continue; + } + + Manifest manifest = GetManifest(nuspecPath, nuspecPropertyProvider, false); + string nupkgPath = GetPackageOutputPath(nuspecPath, manifest, false, false); + Pack(nuspecPath, nupkgPath, manifest, IncludeSymbolsInPackage); + + bool packSymbols = CreateSymbolPackage || CreatePackedPackage; + if (CreateSymbolPackage) + { + Manifest symbolsManifest = GetManifest(nuspecPath, nuspecPropertyProvider, false); + nupkgPath = GetPackageOutputPath(nuspecPath, symbolsManifest, true, false); + Pack(nuspecPath, nupkgPath, symbolsManifest, packSymbols); + } + + if (CreatePackedPackage) + { + Manifest packedManifest = GetManifest(nuspecPath, nuspecPropertyProvider, true); + nupkgPath = GetPackageOutputPath(nuspecPath, packedManifest, false, true); + Pack(nuspecPath, nupkgPath, packedManifest, packSymbols); + } + } + + return !Log.HasLoggedErrors; + } + + private static Func GetNuspecPropertyProviderFunction(ITaskItem[] nuspecProperties) + { + return nuspecProperties == null ? null : NuspecPropertyStringProvider.GetNuspecPropertyProviderFunction(nuspecProperties.Select(p => p.ItemSpec).ToArray()); + } + + private Manifest GetManifest(string nuspecPath, Func nuspecPropertyProvider, bool isPackedPackage) + { + using (var nuspecFile = File.Open(nuspecPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete)) + { + string baseDirectoryPath = (string.IsNullOrEmpty(BaseDirectory)) ? Path.GetDirectoryName(nuspecPath) : BaseDirectory; + Manifest manifest = Manifest.ReadFrom(nuspecFile, nuspecPropertyProvider, false); + + if (isPackedPackage) + { + manifest = TransformManifestToPackedPackageManifest(manifest); + } + return manifest; + } + } + + private string GetPackageOutputPath(string nuspecPath, Manifest manifest, bool isSymbolsPackage, bool applyPrefix) + { + string id = manifest.Metadata.Id; + + if (String.IsNullOrEmpty(id)) + { + Log.LogError($"Nuspec {nuspecPath} does not contain a valid Id"); + return string.Empty; + } + + // Overriding the Version from the Metadata if one gets passed in. + if (!string.IsNullOrEmpty(PackageVersion)) + { + NuGetVersion overrideVersion; + if (NuGetVersion.TryParse(PackageVersion, out overrideVersion)) + { + manifest.Metadata.Version = overrideVersion; + } + else + { + Log.LogError($"Failed to parse Package Version: '{PackageVersion}' is not a valid version."); + } + } + + string version = manifest.Metadata.Version.ToString(); + + if (String.IsNullOrEmpty(version)) + { + Log.LogError($"Nuspec {nuspecPath} does not contain a valid version"); + return string.Empty; + } + + string nupkgOutputDirectory = OutputDirectory; + + if (isSymbolsPackage && !string.IsNullOrEmpty(SymbolPackageOutputDirectory)) + { + nupkgOutputDirectory = SymbolPackageOutputDirectory; + } + + string nupkgExtension = isSymbolsPackage ? _symbolsPackageExtension : _packageExtension; + return Path.Combine(nupkgOutputDirectory, $"{id}.{version}{nupkgExtension}"); + } + + public void Pack(string nuspecPath, string nupkgPath, Manifest manifest, bool packSymbols) + { + bool creatingSymbolsPackage = packSymbols && (Path.GetExtension(nupkgPath) == _symbolsPackageExtension); + try + { + PackageBuilder builder = new PackageBuilder(); + + string baseDirectoryPath = (string.IsNullOrEmpty(BaseDirectory)) ? Path.GetDirectoryName(nuspecPath) : BaseDirectory; + builder.Populate(manifest.Metadata); + builder.PopulateFiles(baseDirectoryPath, manifest.Files); + + if (creatingSymbolsPackage) + { + // For symbols packages, filter out excludes + PathResolver.FilterPackageFiles( + builder.Files, + file => file.Path, + SymbolPackageExcludes); + + // Symbol packages are only valid if they contain both symbols and sources. + Dictionary pathHasMatches = LibPackageExcludes.ToDictionary( + path => path, + path => PathResolver.GetMatches(builder.Files, file => file.Path, new[] { path }).Any()); + + if (!pathHasMatches.Values.Any(i => i)) + { + Log.LogMessage(LogImportance.Low, $"Nuspec {nuspecPath} does not contain symbol or source files. Not creating symbol package."); + return; + } + foreach (var pathPair in pathHasMatches.Where(pathMatchPair => !pathMatchPair.Value)) + { + Log.LogMessage(LogImportance.Low, $"Nuspec {nuspecPath} does not contain any files matching {pathPair.Key}. Not creating symbol package."); + return; + } + } + else if(!packSymbols) + { + // for packages which do not include symbols (not symbols or packed packages), filter lib excludes + PathResolver.FilterPackageFiles( + builder.Files, + file => file.Path, + LibPackageExcludes); + } + + var directory = Path.GetDirectoryName(nupkgPath); + if (!Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + + using (var fileStream = File.Create(nupkgPath)) + { + builder.Save(fileStream); + } + + Log.LogMessage($"Created '{nupkgPath}'"); + } + catch (Exception e) + { + string packageType = "lib"; + if (creatingSymbolsPackage) + { + packageType = "symbol"; + } + else if (packSymbols) + { + packageType = "packed"; + } + Log.LogError($"Error when creating nuget {packageType} package from {nuspecPath}. {e}"); + } + } + + private Manifest TransformManifestToPackedPackageManifest(Manifest manifest) + { + ManifestMetadata manifestMetadata = manifest.Metadata; + + // Update Id + string _packageNamePrefix = PackedPackageNamePrefix != null ? PackedPackageNamePrefix : _defaultPackedPackagePrefix; + manifestMetadata.Id = $"{_packageNamePrefix}.{manifestMetadata.Id}"; + + // Update dependencies + List packedPackageDependencyGroups = new List(); + foreach(var dependencyGroup in manifestMetadata.DependencyGroups) + { + List packages = new List(); + foreach(var dependency in dependencyGroup.Packages) + { + NuGet.Packaging.Core.PackageDependency package = new NuGet.Packaging.Core.PackageDependency($"{_packageNamePrefix}.{dependency.Id}", dependency.VersionRange, dependency.Include, dependency.Exclude); + packages.Add(package); + } + PackageDependencyGroup packageDependencyGroup = new PackageDependencyGroup(dependencyGroup.TargetFramework, packages); + packedPackageDependencyGroups.Add(packageDependencyGroup); + } + manifestMetadata.DependencyGroups = packedPackageDependencyGroups; + + // Update runtime.json + List manifestFiles = new List(); + + foreach(ManifestFile file in manifest.Files) + { + string fileName = file.Source; + if(Path.GetFileName(fileName) == "runtime.json" && file.Target == "") + { + string packedPackageSourcePath = Path.Combine(Path.GetDirectoryName(fileName), string.Join(".", _packageNamePrefix, Path.GetFileName(fileName))); + file.Source = File.Exists(packedPackageSourcePath) ? packedPackageSourcePath : fileName; + file.Target = "runtime.json"; + } + manifestFiles.Add(file); + } + Manifest packedPackageManifest = new Manifest(manifestMetadata, manifestFiles); + return manifest; + } + + private IEnumerable LibPackageExcludes + { + get + { + return _libPackageExcludes + .Concat(AdditionalLibPackageExcludes?.Select(item => item.ItemSpec) ?? Enumerable.Empty()); + } + } + + private IEnumerable SymbolPackageExcludes + { + get + { + return _symbolPackageExcludes + .Concat(AdditionalSymbolPackageExcludes?.Select(item => item.ItemSpec) ?? Enumerable.Empty()); + } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/NuspecPropertyStringProvider.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/NuspecPropertyStringProvider.cs new file mode 100644 index 000000000..d10fad1b2 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/NuspecPropertyStringProvider.cs @@ -0,0 +1,72 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Collections.Generic; + +namespace Microsoft.DotNet.Build.Tasks.Packaging +{ + public class NuspecPropertyStringProvider + { + public static Dictionary GetNuspecPropertyDictionary(string[] nuspecProperties) + { + if (nuspecProperties == null) + { + return null; + } + + var propertyDictionary = new Dictionary(); + foreach (var propertyString in nuspecProperties) + { + var property = GetKeyValuePair(propertyString); + propertyDictionary[property.Item1] = property.Item2; + } + + return propertyDictionary; + } + + public static Func GetNuspecPropertyProviderFunction(string[] nuspecPropertyStrings) + { + var propertyDictionary = GetNuspecPropertyDictionary(nuspecPropertyStrings); + + if (propertyDictionary == null) + { + return null; + } + + return k => propertyDictionary[k]; + } + + private static Tuple GetKeyValuePair(string propertyString) + { + propertyString = propertyString.Trim(); + + var indexOfEquals = propertyString.IndexOf("=", StringComparison.Ordinal); + + if (indexOfEquals == -1) + { + throw new InvalidDataException($"Nuspec property {propertyString} does not have an \'=\' character in it"); + } + + if (indexOfEquals == propertyString.Length - 1) + { + throw new InvalidDataException($"Nuspec property {propertyString} does not have a value"); + } + + if (indexOfEquals == 0) + { + throw new InvalidDataException($"Nuspec property {propertyString} does not have a key"); + } + + var key = propertyString.Substring(0, indexOfEquals); + + var valueStartIndex = indexOfEquals + 1; + var valueLength = propertyString.Length - valueStartIndex; + var value = propertyString.Substring(valueStartIndex, valueLength); + + return new Tuple(key, value); + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/PackagingTask.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/PackagingTask.cs new file mode 100644 index 000000000..24e2e0047 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/PackagingTask.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks.Packaging +{ + public abstract partial class PackagingTask : ITask + { + private Log _log = null; + + internal Log Log + { + get { return _log ?? (_log = new Log(new TaskLoggingHelper(this))); } + } + + public PackagingTask() + { + } + + public IBuildEngine BuildEngine + { + get; + set; + } + + public ITaskHost HostObject + { + get; + set; + } + + public abstract bool Execute(); + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReadNuGetPackageInfos.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReadNuGetPackageInfos.cs new file mode 100644 index 000000000..43a04ebd0 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReadNuGetPackageInfos.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks +{ + public class ReadNuGetPackageInfos : Task + { + [Required] + public string[] PackagePaths { get; set; } + + /// + /// %(Identity): Path to the original nupkg. + /// %(PackageId): Identity of the package. + /// %(PackageVersion): Version of the package. + /// + [Output] + public ITaskItem[] PackageInfoItems { get; set; } + + public override bool Execute() + { + PackageInfoItems = PackagePaths + .Select(p => + { + PackageIdentity identity = ReadIdentity(p); + return new TaskItem( + p, + new Dictionary + { + ["PackageId"] = identity.Id, + ["PackageVersion"] = identity.Version.OriginalVersion + }); + }) + .ToArray(); + + return !Log.HasLoggedErrors; + } + + public static PackageIdentity ReadIdentity(string nupkgFile) + { + using (var reader = new PackageArchiveReader(nupkgFile)) + { + return reader.GetIdentity(); + } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/RemoveInternetSourcesFromNuGetConfig.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/RemoveInternetSourcesFromNuGetConfig.cs new file mode 100755 index 000000000..65aa69ac8 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/RemoveInternetSourcesFromNuGetConfig.cs @@ -0,0 +1,83 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + /* + * This task removes internet sources from a given NuGet.config. In the offline build mode, it removes all + * feeds that begin with http or https. In the online build mode, it removes only the internal dnceng feeds that + * source-build does not have access to. + */ + public class RemoveInternetSourcesFromNuGetConfig : Task + { + [Required] + public string NuGetConfigFile { get; set; } + + /// + /// Whether to work in offline mode (remove all internet sources) or online mode (remove only authenticated sources) + /// + public bool OfflineBuild { get; set; } + + /// + /// A list of prefix strings that make the task keep a package source unconditionally. For + /// example, a source named 'darc-pub-dotnet-aspnetcore-e81033e' will be kept if the prefix + /// 'darc-pub-dotnet-aspnetcore-' is in this list. + /// + public string[] KeepFeedPrefixes { get; set; } + + public override bool Execute() + { + XDocument d = XDocument.Load(NuGetConfigFile); + XElement packageSourcesElement = d.Root.Descendants().First(e => e.Name == "packageSources"); + XElement disabledPackageSourcesElement = d.Root.Descendants().FirstOrDefault(e => e.Name == "disabledPackageSources"); + + IEnumerable local = packageSourcesElement.Descendants().Where(e => + { + if (e.Name == "add") + { + string feedName = e.Attribute("key").Value; + if (KeepFeedPrefixes + ?.Any(prefix => feedName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + == true) + { + return true; + } + + string feedUrl = e.Attribute("value").Value; + if (OfflineBuild) + { + return !(feedUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || feedUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase)); + } + else + { + return !( feedUrl.StartsWith("https://pkgs.dev.azure.com/dnceng/_packaging", StringComparison.OrdinalIgnoreCase) || + feedUrl.StartsWith("https://pkgs.dev.azure.com/dnceng/internal/_packaging", StringComparison.OrdinalIgnoreCase) ); + } + } + + return true; + }); + + packageSourcesElement.ReplaceNodes(local.ToArray()); + + // Remove disabledPackageSources element so if any internal packages remain, they are used in source-build + disabledPackageSourcesElement?.ReplaceNodes(new XElement("clear")); + + using (FileStream fs = new FileStream(NuGetConfigFile, FileMode.Create, FileAccess.ReadWrite)) + { + d.Save(fs); + } + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceFeedsInNuGetConfig.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceFeedsInNuGetConfig.cs new file mode 100644 index 000000000..a4519ead5 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceFeedsInNuGetConfig.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + /// + /// Replaces feeds in a NuGet.Config file given a mapping + /// of old feeds to new feeds. + /// + public class ReplaceFeedsInNuGetConfig : Task + { + /// + /// The NuGet.Config file in which to replace feeds. + /// + [Required] + public string InputFile { get; set; } + + /// + /// An item group of feeds to update. + /// %(Identity): The feed URL to find in the NuGet.Config. + /// %(NewFeed): The feed URL to replace %(Identity) with. + /// + [Required] + public ITaskItem[] FeedMapping { get; set; } + + public override bool Execute() + { + string fileContents = File.ReadAllText(InputFile); + bool updated = false; + + foreach (var feed in FeedMapping) + { + string oldFeed = feed.ItemSpec; + string newFeed = feed.GetMetadata("NewFeed"); + + if (fileContents.Contains(oldFeed)) + { + fileContents = fileContents.Replace(oldFeed, newFeed); + updated = true; + } + } + + if (updated) File.WriteAllText(InputFile, fileContents); + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceRegexInFiles.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceRegexInFiles.cs new file mode 100644 index 000000000..9b0d2e10c --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceRegexInFiles.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Text.RegularExpressions; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class ReplaceRegexInFiles : Task + { + [Required] + public string[] InputFiles { get; set; } + + [Required] + public string OldTextRegex { get; set; } + + [Required] + public string NewText { get; set; } + + public override bool Execute() + { + Log.LogMessage($"Replacing '{OldTextRegex}' with '{NewText}'"); + foreach (string file in InputFiles) + { + string fileContents = File.ReadAllText(file); + + fileContents = Regex.Replace(fileContents, OldTextRegex, NewText); + + File.WriteAllText(file, fileContents); + } + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceTextInFile.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceTextInFile.cs new file mode 100644 index 000000000..32b8cdea1 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceTextInFile.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class ReplaceTextInFile : Task + { + [Required] + public string InputFile { get; set; } + + [Required] + public string OldText { get; set; } + + [Required] + public string NewText { get; set; } + + + public override bool Execute() + { + string fileContents = File.ReadAllText(InputFile); + + fileContents = fileContents.Replace(OldText, NewText); + + File.WriteAllText(InputFile, fileContents); + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceTextInFiles.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceTextInFiles.cs new file mode 100644 index 000000000..30f1d6e67 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceTextInFiles.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class ReplaceTextInFiles : Task + { + [Required] + public string[] InputFiles { get; set; } + + [Required] + public string OldText { get; set; } + + [Required] + public string NewText { get; set; } + + public override bool Execute() + { + foreach (string file in InputFiles) + { + string fileContents = File.ReadAllText(file); + + fileContents = fileContents.Replace(OldText, NewText, StringComparison.Ordinal); + + File.WriteAllText(file, fileContents); + } + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/RepoTasks/JoinItems.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/RepoTasks/JoinItems.cs new file mode 100644 index 000000000..55f941c16 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/RepoTasks/JoinItems.cs @@ -0,0 +1,139 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +// Copied from https://github.com/aspnet/Universe/blob/1f8f30a1e834eff147ced0c669cef8828f9511c8/build/tasks/JoinItems.cs. +// When this task is available in https://github.com/dotnet/Arcade, switch to use that version. +// Modified to allow multiple Right matches using GroupJoin. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System; +using System.Linq; + +namespace RepoTasks +{ + public class JoinItems : Task + { + [Required] + public ITaskItem[] Left { get; set; } + + [Required] + public ITaskItem[] Right { get; set; } + + // The metadata to use as the new item spec. If not specified, LeftKey is used. + public string LeftItemSpec { get; set; } + + // LeftKey and RightKey: The metadata to join on. If not set, then use the ItemSpec + public string LeftKey { get; set; } + + public string RightKey { get; set; } + + + // LeftMetadata and RightMetadata: The metadata names to include in the result. Specify "*" to include all metadata + public string[] LeftMetadata { get; set; } + + public string[] RightMetadata { get; set; } + + + [Output] + public ITaskItem[] JoinResult { get; private set; } + + public override bool Execute() + { + bool useAllLeftMetadata = LeftMetadata != null && LeftMetadata.Length == 1 && LeftMetadata[0] == "*"; + bool useAllRightMetadata = RightMetadata != null && RightMetadata.Length == 1 && RightMetadata[0] == "*"; + var newItemSpec = string.IsNullOrEmpty(LeftItemSpec) + ? LeftKey + : LeftItemSpec; + + JoinResult = Left.GroupJoin(Right, + item => GetKeyValue(LeftKey, item), + item => GetKeyValue(RightKey, item), + (left, rights) => + { + // If including all metadata from left items and none from right items, just return left items directly + if (useAllLeftMetadata && + string.IsNullOrEmpty(LeftKey) && + string.IsNullOrEmpty(LeftItemSpec) && + (RightMetadata == null || RightMetadata.Length == 0)) + { + return left; + } + + // If including all metadata from all right items and none from left items, just return the right items directly + if (useAllRightMetadata && + string.IsNullOrEmpty(RightKey) && + string.IsNullOrEmpty(LeftItemSpec) && + (LeftMetadata == null || LeftMetadata.Length == 0)) + { + return rights.Aggregate( + new TaskItem(), + (agg, next) => + { + CopyAllMetadata(next, agg); + return agg; + }); + } + + var ret = new TaskItem(GetKeyValue(newItemSpec, left)); + + // Weird ordering here is to prefer left metadata in all cases, as CopyToMetadata doesn't overwrite any existing metadata + if (useAllLeftMetadata) + { + CopyAllMetadata(left, ret); + } + + if (!useAllRightMetadata && RightMetadata != null) + { + foreach (string name in RightMetadata) + { + foreach (var right in rights) + { + ret.SetMetadata(name, right.GetMetadata(name)); + } + } + } + + if (!useAllLeftMetadata && LeftMetadata != null) + { + foreach (string name in LeftMetadata) + { + ret.SetMetadata(name, left.GetMetadata(name)); + } + } + + if (useAllRightMetadata) + { + foreach (var right in rights) + { + CopyAllMetadata(right, ret); + } + } + + return (ITaskItem)ret; + }, + StringComparer.OrdinalIgnoreCase).ToArray(); + + return true; + } + + static void CopyAllMetadata(ITaskItem source, ITaskItem dest) + { + // CopyMetadata adds an OriginalItemSpec, which we don't want. So we subsequently remove it + source.CopyMetadataTo(dest); + dest.RemoveMetadata("OriginalItemSpec"); + } + + static string GetKeyValue(string key, ITaskItem item) + { + if (string.IsNullOrEmpty(key)) + { + return item.ItemSpec; + } + else + { + return item.GetMetadata(key); + } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateJson.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateJson.cs new file mode 100644 index 000000000..9820e4af2 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateJson.cs @@ -0,0 +1,69 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Linq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + // Takes a path to a path to a json file and a + // string that represents a dotted path to an attribute + // and updates that attribute with the new value provided. + public class UpdateJson : Task + { + [Required] + public string JsonFilePath { get; set; } + + [Required] + public string PathToAttribute { get; set; } + + [Required] + public string NewAttributeValue { get; set; } + + public bool SkipUpdateIfMissingKey { get; set; } + + public override bool Execute() + { + JObject jsonObj = JObject.Parse(File.ReadAllText(JsonFilePath)); + + string[] escapedPathToAttributeParts = PathToAttribute.Replace("\\.", "\x1F").Split('.'); + for (int i = 0; i < escapedPathToAttributeParts.Length; ++i) + { + escapedPathToAttributeParts[i] = escapedPathToAttributeParts[i].Replace("\x1F", "."); + } + UpdateAttribute(jsonObj, escapedPathToAttributeParts, NewAttributeValue); + + File.WriteAllText(JsonFilePath, jsonObj.ToString()); + return true; + } + + private void UpdateAttribute(JToken jsonObj, string[] path, string newValue) + { + string pathItem = path[0]; + if (jsonObj[pathItem] == null) + { + string message = $"Path item [{nameof(PathToAttribute)}] not found in json file."; + if (SkipUpdateIfMissingKey) + { + Log.LogMessage(MessageImportance.Low, $"Skipping update: {message} {pathItem}"); + return; + } + throw new ArgumentException(message, pathItem); + } + + if (path.Length == 1) + { + jsonObj[pathItem] = newValue; + return; + } + + UpdateAttribute(jsonObj[pathItem], path.Skip(1).ToArray(), newValue); + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadClient.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadClient.cs new file mode 100644 index 000000000..c02055705 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadClient.cs @@ -0,0 +1,285 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class UploadClient + { + private TaskLoggingHelper log; + + public UploadClient(TaskLoggingHelper loggingHelper) + { + log = loggingHelper; + } + + public string EncodeBlockIds(int numberOfBlocks, int lengthOfId) + { + string numberOfBlocksString = numberOfBlocks.ToString("D" + lengthOfId); + if (Encoding.UTF8.GetByteCount(numberOfBlocksString) <= 64) + { + byte[] bytes = Encoding.UTF8.GetBytes(numberOfBlocksString); + return Convert.ToBase64String(bytes); + } + else + { + throw new Exception("Task failed - Could not encode block id."); + } + } + + public async Task UploadBlockBlobAsync( + CancellationToken ct, + string AccountName, + string AccountKey, + string ContainerName, + string filePath, + string destinationBlob, + string contentType, + int uploadTimeout, + string leaseId = "") + { + string resourceUrl = AzureHelper.GetContainerRestUrl(AccountName, ContainerName); + + string fileName = destinationBlob; + fileName = fileName.Replace("\\", "/"); + string blobUploadUrl = resourceUrl + "/" + fileName; + int size = (int)new FileInfo(filePath).Length; + int blockSize = 4 * 1024 * 1024; //4MB max size of a block blob + int bytesLeft = size; + List blockIds = new List(); + int numberOfBlocks = (size / blockSize) + 1; + int countForId = 0; + using (FileStream fileStreamTofilePath = new FileStream(filePath, FileMode.Open, FileAccess.Read)) + { + int offset = 0; + + while (bytesLeft > 0) + { + int nextBytesToRead = (bytesLeft < blockSize) ? bytesLeft : blockSize; + byte[] fileBytes = new byte[blockSize]; + int read = fileStreamTofilePath.Read(fileBytes, 0, nextBytesToRead); + + if (nextBytesToRead != read) + { + throw new Exception(string.Format( + "Number of bytes read ({0}) from file {1} isn't equal to the number of bytes expected ({2}) .", + read, fileName, nextBytesToRead)); + } + + string blockId = EncodeBlockIds(countForId, numberOfBlocks.ToString().Length); + + blockIds.Add(blockId); + string blockUploadUrl = blobUploadUrl + "?comp=block&blockid=" + WebUtility.UrlEncode(blockId); + + using (HttpClient client = new HttpClient()) + { + client.DefaultRequestHeaders.Clear(); + + // In random occassions the request fails if the network is slow and it takes more than 100 seconds to upload 4MB. + client.Timeout = TimeSpan.FromMinutes(uploadTimeout); + Func createRequest = () => + { + DateTime dt = DateTime.UtcNow; + var req = new HttpRequestMessage(HttpMethod.Put, blockUploadUrl); + req.Headers.Add( + AzureHelper.DateHeaderString, + dt.ToString("R", CultureInfo.InvariantCulture)); + req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); + if (!string.IsNullOrWhiteSpace(leaseId)) + { + log.LogMessage($"Sending request: {leaseId} {blockUploadUrl}"); + req.Headers.Add("x-ms-lease-id", leaseId); + } + req.Headers.Add( + AzureHelper.AuthorizationHeaderString, + AzureHelper.AuthorizationHeader( + AccountName, + AccountKey, + "PUT", + dt, + req, + string.Empty, + string.Empty, + nextBytesToRead.ToString(), + string.Empty)); + + Stream postStream = new MemoryStream(); + postStream.Write(fileBytes, 0, nextBytesToRead); + postStream.Seek(0, SeekOrigin.Begin); + req.Content = new StreamContent(postStream); + return req; + }; + + log.LogMessage(MessageImportance.Low, "Sending request to upload part {0} of file {1}", countForId, fileName); + + using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest)) + { + log.LogMessage( + MessageImportance.Low, + "Received response to upload part {0} of file {1}: Status Code:{2} Status Desc: {3}", + countForId, + fileName, + response.StatusCode, + await response.Content.ReadAsStringAsync()); + } + } + + offset += read; + bytesLeft -= nextBytesToRead; + countForId += 1; + } + } + + string blockListUploadUrl = blobUploadUrl + "?comp=blocklist"; + + using (HttpClient client = new HttpClient()) + { + Func createRequest = () => + { + DateTime dt1 = DateTime.UtcNow; + var req = new HttpRequestMessage(HttpMethod.Put, blockListUploadUrl); + req.Headers.Add(AzureHelper.DateHeaderString, dt1.ToString("R", CultureInfo.InvariantCulture)); + req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); + if (string.IsNullOrEmpty(contentType)) + { + contentType = DetermineContentTypeBasedOnFileExtension(filePath); + } + if (!string.IsNullOrEmpty(contentType)) + { + req.Headers.Add(AzureHelper.ContentTypeString, contentType); + } + string cacheControl = DetermineCacheControlBasedOnFileExtension(filePath); + if (!string.IsNullOrEmpty(cacheControl)) + { + req.Headers.Add(AzureHelper.CacheControlString, cacheControl); + } + + var body = new StringBuilder(""); + foreach (object item in blockIds) + body.AppendFormat("{0}", item); + + body.Append(""); + byte[] bodyData = Encoding.UTF8.GetBytes(body.ToString()); + if (!string.IsNullOrWhiteSpace(leaseId)) + { + log.LogMessage($"Sending list request: {leaseId} {blockListUploadUrl}"); + req.Headers.Add("x-ms-lease-id", leaseId); + } + req.Headers.Add( + AzureHelper.AuthorizationHeaderString, + AzureHelper.AuthorizationHeader( + AccountName, + AccountKey, + "PUT", + dt1, + req, + string.Empty, + string.Empty, + bodyData.Length.ToString(), + string.Empty)); + + Stream postStream = new MemoryStream(); + postStream.Write(bodyData, 0, bodyData.Length); + postStream.Seek(0, SeekOrigin.Begin); + req.Content = new StreamContent(postStream); + return req; + }; + + using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest)) + { + log.LogMessage( + MessageImportance.Low, + "Received response to combine block list for file {0}: Status Code:{1} Status Desc: {2}", + fileName, + response.StatusCode, + await response.Content.ReadAsStringAsync()); + } + } + } + + public async Task FileEqualsExistingBlobAsync( + string accountName, + string accountKey, + string containerName, + string filePath, + string destinationBlob, + int uploadTimeout) + { + using (var client = new HttpClient + { + Timeout = TimeSpan.FromMinutes(uploadTimeout) + }) + { + log.LogMessage( + MessageImportance.Low, + $"Downloading blob {destinationBlob} to check if identical."); + + string blobUrl = AzureHelper.GetBlobRestUrl(accountName, containerName, destinationBlob); + var createRequest = AzureHelper.RequestMessage("GET", blobUrl, accountName, accountKey); + + using (HttpResponseMessage response = await AzureHelper.RequestWithRetry( + log, + client, + createRequest)) + { + if (!response.IsSuccessStatusCode) + { + throw new HttpRequestException( + $"Failed to retrieve existing blob {destinationBlob}, " + + $"status code {response.StatusCode}."); + } + + byte[] existingBytes = await response.Content.ReadAsByteArrayAsync(); + byte[] localBytes = File.ReadAllBytes(filePath); + + bool equal = localBytes.SequenceEqual(existingBytes); + + if (equal) + { + log.LogMessage( + MessageImportance.Normal, + "Item exists in blob storage, and is verified to be identical. " + + $"File: '{filePath}' Blob: '{destinationBlob}'"); + } + + return equal; + } + } + } + + private string DetermineContentTypeBasedOnFileExtension(string filename) + { + if (Path.GetExtension(filename) == ".svg") + { + return "image/svg+xml"; + } + else if (Path.GetExtension(filename) == ".version") + { + return "text/plain"; + } + return string.Empty; + } + private string DetermineCacheControlBasedOnFileExtension(string filename) + { + if (Path.GetExtension(filename) == ".svg") + { + return "No-Cache"; + } + return string.Empty; + } + } +} \ No newline at end of file diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadToAzure.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadToAzure.cs new file mode 100644 index 000000000..5431194d6 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadToAzure.cs @@ -0,0 +1,208 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using System.Xml; +using ThreadingTask = System.Threading.Tasks.Task; + +namespace Microsoft.DotNet.Build.Tasks +{ + + public class UploadToAzure : AzureConnectionStringBuildTask, ICancelableTask + { + private static readonly CancellationTokenSource TokenSource = new CancellationTokenSource(); + private static readonly CancellationToken CancellationToken = TokenSource.Token; + + /// + /// The name of the container to access. The specified name must be in the correct format, see the + /// following page for more info. https://msdn.microsoft.com/en-us/library/azure/dd135715.aspx + /// + [Required] + public string ContainerName { get; set; } + + /// + /// An item group of files to upload. Each item must have metadata RelativeBlobPath + /// that specifies the path relative to ContainerName where the item will be uploaded. + /// + [Required] + public ITaskItem[] Items { get; set; } + + /// + /// Indicates if the destination blob should be overwritten if it already exists. The default if false. + /// + public bool Overwrite { get; set; } = false; + + /// + /// Enables idempotency when Overwrite is false. + /// + /// false: (default) Attempting to upload an item that already exists fails. + /// + /// true: When an item already exists, download the existing blob to check if it's + /// byte-for-byte identical to the one being uploaded. If so, pass. If not, fail. + /// + public bool PassIfExistingItemIdentical { get; set; } + + /// + /// Specifies the maximum number of clients to concurrently upload blobs to azure + /// + public int MaxClients { get; set; } = 8; + + public int UploadTimeoutInMinutes { get; set; } = 5; + + public void Cancel() + { + TokenSource.Cancel(); + } + + public override bool Execute() + { + return ExecuteAsync(CancellationToken).GetAwaiter().GetResult(); + } + + public async Task ExecuteAsync(CancellationToken ct) + { + ParseConnectionString(); + // If the connection string AND AccountKey & AccountName are provided, error out. + if (Log.HasLoggedErrors) + { + return false; + } + + Log.LogMessage( + MessageImportance.Normal, + "Begin uploading blobs to Azure account {0} in container {1}.", + AccountName, + ContainerName); + + if (Items.Length == 0) + { + Log.LogError("No items were provided for upload."); + return false; + } + + // first check what blobs are present + string checkListUrl = $"{AzureHelper.GetContainerRestUrl(AccountName, ContainerName)}?restype=container&comp=list"; + + HashSet blobsPresent = new HashSet(StringComparer.OrdinalIgnoreCase); + + try + { + using (HttpClient client = new HttpClient()) + { + var createRequest = AzureHelper.RequestMessage("GET", checkListUrl, AccountName, AccountKey); + + Log.LogMessage(MessageImportance.Low, "Sending request to check whether Container blobs exist"); + using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) + { + var doc = new XmlDocument(); + doc.LoadXml(await response.Content.ReadAsStringAsync()); + + XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob"); + + foreach (XmlNode node in nodes) + { + blobsPresent.Add(node["Name"].InnerText); + } + + Log.LogMessage(MessageImportance.Low, "Received response to check whether Container blobs exist"); + } + } + + using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients)) + { + await ThreadingTask.WhenAll(Items.Select(item => UploadAsync(ct, item, blobsPresent, clientThrottle))); + } + + Log.LogMessage(MessageImportance.Normal, "Upload to Azure is complete, a total of {0} items were uploaded.", Items.Length); + } + catch (Exception e) + { + Log.LogErrorFromException(e, true); + } + return !Log.HasLoggedErrors; + } + + private async ThreadingTask UploadAsync(CancellationToken ct, ITaskItem item, HashSet blobsPresent, SemaphoreSlim clientThrottle) + { + if (ct.IsCancellationRequested) + { + Log.LogError("Task UploadToAzure cancelled"); + ct.ThrowIfCancellationRequested(); + } + + string relativeBlobPath = item.GetMetadata("RelativeBlobPath"); + if (string.IsNullOrEmpty(relativeBlobPath)) + throw new Exception(string.Format("Metadata 'RelativeBlobPath' is missing for item '{0}'.", item.ItemSpec)); + + if (!File.Exists(item.ItemSpec)) + throw new Exception(string.Format("The file '{0}' does not exist.", item.ItemSpec)); + + UploadClient uploadClient = new UploadClient(Log); + + if (!Overwrite && blobsPresent.Contains(relativeBlobPath)) + { + if (PassIfExistingItemIdentical && + await ItemEqualsExistingBlobAsync(item, relativeBlobPath, uploadClient, clientThrottle)) + { + return; + } + + throw new Exception(string.Format("The blob '{0}' already exists.", relativeBlobPath)); + } + + string contentType = item.GetMetadata("ContentType"); + + await clientThrottle.WaitAsync(); + + try + { + Log.LogMessage("Uploading {0} to {1}.", item.ItemSpec, ContainerName); + await + uploadClient.UploadBlockBlobAsync( + ct, + AccountName, + AccountKey, + ContainerName, + item.ItemSpec, + relativeBlobPath, + contentType, + UploadTimeoutInMinutes); + } + finally + { + clientThrottle.Release(); + } + } + + private async Task ItemEqualsExistingBlobAsync( + ITaskItem item, + string relativeBlobPath, + UploadClient client, + SemaphoreSlim clientThrottle) + { + await clientThrottle.WaitAsync(); + try + { + return await client.FileEqualsExistingBlobAsync( + AccountName, + AccountKey, + ContainerName, + item.ItemSpec, + relativeBlobPath, + UploadTimeoutInMinutes); + } + finally + { + clientThrottle.Release(); + } + } + } +} \ No newline at end of file diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/AnnotatedUsage.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/AnnotatedUsage.cs new file mode 100644 index 000000000..e15a032f6 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/AnnotatedUsage.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport +{ + public class AnnotatedUsage + { + public Usage Usage { get; set; } + + public string Project { get; set; } + public string SourceBuildPackageIdCreator { get; set; } + public string ProdConPackageIdCreator { get; set; } + public bool EndsUpInOutput { get; set; } + public bool TestProjectByHeuristic { get; set; } + public bool TestProjectOnlyByHeuristic { get; set; } + public bool IsDirectDependency { get; set; } + public bool IsAutoReferenced { get; set; } + + public XElement ToXml() => new XElement( + nameof(AnnotatedUsage), + Usage.ToXml().Attributes(), + Project.ToXAttributeIfNotNull(nameof(Project)), + SourceBuildPackageIdCreator.ToXAttributeIfNotNull(nameof(SourceBuildPackageIdCreator)), + ProdConPackageIdCreator.ToXAttributeIfNotNull(nameof(ProdConPackageIdCreator)), + TestProjectByHeuristic.ToXAttributeIfTrue(nameof(TestProjectByHeuristic)), + TestProjectOnlyByHeuristic.ToXAttributeIfTrue(nameof(TestProjectOnlyByHeuristic)), + IsDirectDependency.ToXAttributeIfTrue(nameof(IsDirectDependency)), + IsAutoReferenced.ToXAttributeIfTrue(nameof(IsAutoReferenced)), + EndsUpInOutput.ToXAttributeIfTrue(nameof(EndsUpInOutput))); + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/Usage.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/Usage.cs new file mode 100644 index 000000000..8b35947db --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/Usage.cs @@ -0,0 +1,112 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using NuGet.Packaging.Core; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport +{ + public class Usage : IEquatable + { + public PackageIdentity PackageIdentity { get; set; } + + public string AssetsFile { get; set; } + + public bool IsDirectDependency { get; set; } + + public bool IsAutoReferenced { get; set; } + + /// + /// The Runtime ID this package is for, or null. Runtime packages (are assumed to) have the + /// id 'runtime.{rid}.{rest of id}'. We can't use a simple regex to grab this value since + /// the RID may have '.' in it, so this is saved in the build context where possible RIDs + /// are read from Microsoft.NETCore.Platforms. + /// + public string RuntimePackageRid { get; set; } + + public XElement ToXml() => new XElement( + nameof(Usage), + PackageIdentity.ToXElement().Attributes(), + AssetsFile.ToXAttributeIfNotNull("File"), + IsDirectDependency.ToXAttributeIfTrue(nameof(IsDirectDependency)), + IsAutoReferenced.ToXAttributeIfTrue(nameof(IsAutoReferenced)), + RuntimePackageRid.ToXAttributeIfNotNull("Rid")); + + public static Usage Parse(XElement xml) => new Usage + { + PackageIdentity = XmlParsingHelpers.ParsePackageIdentity(xml), + AssetsFile = xml.Attribute("File")?.Value, + IsDirectDependency = Convert.ToBoolean(xml.Attribute(nameof(IsDirectDependency))?.Value), + IsAutoReferenced = Convert.ToBoolean(xml.Attribute(nameof(IsAutoReferenced))?.Value), + RuntimePackageRid = xml.Attribute("Rid")?.Value + }; + + public static Usage Create( + string assetsFile, + PackageIdentity identity, + bool isDirectDependency, + bool isAutoReferenced, + IEnumerable possibleRuntimePackageRids) + { + return new Usage + { + AssetsFile = assetsFile, + PackageIdentity = identity, + IsDirectDependency = isDirectDependency, + IsAutoReferenced = isAutoReferenced, + RuntimePackageRid = possibleRuntimePackageRids + .Where(rid => identity.Id.StartsWith($"runtime.{rid}.", StringComparison.Ordinal)) + .OrderByDescending(rid => rid.Length) + .FirstOrDefault() + }; + } + + public PackageIdentity GetIdentityWithoutRid() + { + if (!string.IsNullOrEmpty(RuntimePackageRid)) + { + string prefix = $"runtime.{RuntimePackageRid}."; + if (PackageIdentity.Id.StartsWith(prefix, StringComparison.Ordinal)) + { + return new PackageIdentity( + PackageIdentity.Id + .Remove(0, prefix.Length) + .Insert(0, "runtime.placeholder-rid."), + PackageIdentity.Version); + } + } + return PackageIdentity; + } + + public override string ToString() => + $"{PackageIdentity.Id} {PackageIdentity.Version} " + + (string.IsNullOrEmpty(RuntimePackageRid) ? "" : $"({RuntimePackageRid}) ") + + (string.IsNullOrEmpty(AssetsFile) ? "with unknown use" : $"used by '{AssetsFile}'"); + + public bool Equals(Usage other) + { + if (other is null) + { + return false; + } + if (ReferenceEquals(this, other)) + { + return true; + } + return + Equals(PackageIdentity, other.PackageIdentity) && + string.Equals(AssetsFile, other.AssetsFile, StringComparison.Ordinal) && + string.Equals(RuntimePackageRid, other.RuntimePackageRid, StringComparison.Ordinal); + } + + public override int GetHashCode() => ( + PackageIdentity, + AssetsFile, + RuntimePackageRid + ).GetHashCode(); + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/UsageData.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/UsageData.cs new file mode 100644 index 000000000..ce6c836c4 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/UsageData.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using NuGet.Packaging.Core; +using System.Linq; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport +{ + public class UsageData + { + public string CreatedByRid { get; set; } + public string[] ProjectDirectories { get; set; } + public PackageIdentity[] NeverRestoredTarballPrebuilts { get; set; } + public Usage[] Usages { get; set; } + + public XElement ToXml() => new XElement( + nameof(UsageData), + CreatedByRid == null ? null : new XElement( + nameof(CreatedByRid), + CreatedByRid), + ProjectDirectories?.Any() != true ? null : new XElement( + nameof(ProjectDirectories), + ProjectDirectories + .Select(dir => new XElement("Dir", dir))), + NeverRestoredTarballPrebuilts?.Any() != true ? null : new XElement( + nameof(NeverRestoredTarballPrebuilts), + NeverRestoredTarballPrebuilts + .OrderBy(id => id) + .Select(id => id.ToXElement())), + Usages?.Any() != true ? null : new XElement( + nameof(Usages), + Usages + .OrderBy(u => u.PackageIdentity) + .ThenByOrdinal(u => u.AssetsFile) + .Select(u => u.ToXml()))); + + public static UsageData Parse(XElement xml) => new UsageData + { + CreatedByRid = xml.Element(nameof(CreatedByRid)) + ?.Value, + ProjectDirectories = xml.Element(nameof(ProjectDirectories)) == null ? new string[] { } : + xml.Element(nameof(ProjectDirectories)).Elements() + .Select(x => x.Value) + .ToArray(), + NeverRestoredTarballPrebuilts = xml.Element(nameof(NeverRestoredTarballPrebuilts)) == null ? new PackageIdentity[] { } : + xml.Element(nameof(NeverRestoredTarballPrebuilts)).Elements() + .Select(XmlParsingHelpers.ParsePackageIdentity) + .ToArray(), + Usages = xml.Element(nameof(Usages)) == null ? new Usage[] { } : + xml.Element(nameof(Usages)).Elements() + .Select(Usage.Parse) + .ToArray() + }; + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/ValidateUsageAgainstBaseline.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/ValidateUsageAgainstBaseline.cs new file mode 100644 index 000000000..d96dbf8d0 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/ValidateUsageAgainstBaseline.cs @@ -0,0 +1,141 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Packaging.Core; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport +{ + public class ValidateUsageAgainstBaseline : Task + { + [Required] + public string DataFile { get; set; } + + [Required] + public string BaselineDataFile { get; set; } + + [Required] + public string OutputBaselineFile { get; set; } + + [Required] + public string OutputReportFile { get; set; } + + public bool AllowTestProjectUsage { get; set; } + + public override bool Execute() + { + var used = UsageData.Parse(XElement.Parse(File.ReadAllText(DataFile))); + var baseline = UsageData.Parse(XElement.Parse(File.ReadAllText(BaselineDataFile))); + + Comparison diff = Compare( + used.Usages.Select(u => u.GetIdentityWithoutRid()).Distinct(), + baseline.Usages.Select(u => u.GetIdentityWithoutRid()).Distinct()); + + var report = new XElement("BaselineComparison"); + + bool tellUserToUpdateBaseline = false; + + if (diff.Added.Any()) + { + tellUserToUpdateBaseline = true; + Log.LogError( + $"{diff.Added.Length} new packages used not in baseline! See report " + + $"at {OutputReportFile} for more information. Package IDs are:\n" + + string.Join("\n", diff.Added.Select(u => u.ToString()))); + + // In the report, list full usage info, not only identity. + report.Add( + new XElement( + "New", + used.Usages + .Where(u => diff.Added.Contains(u.GetIdentityWithoutRid())) + .Select(u => u.ToXml()))); + } + if (diff.Removed.Any()) + { + tellUserToUpdateBaseline = true; + Log.LogMessage( + MessageImportance.High, + $"{diff.Removed.Length} packages in baseline weren't used!"); + + report.Add(new XElement("Removed", diff.Removed.Select(id => id.ToXElement()))); + } + if (diff.Unchanged.Any()) + { + Log.LogMessage( + MessageImportance.High, + $"{diff.Unchanged.Length} packages used as expected in the baseline."); + } + + if (!AllowTestProjectUsage) + { + Usage[] testProjectUsages = used.Usages + .Where(WriteUsageReports.IsTestUsageByHeuristic) + .ToArray(); + + if (testProjectUsages.Any()) + { + string[] projects = testProjectUsages + .Select(u => u.AssetsFile) + .Distinct() + .ToArray(); + + Log.LogError( + $"{testProjectUsages.Length} forbidden test usages found in " + + $"{projects.Length} projects:\n" + + string.Join("\n", projects)); + } + } + + // Simplify the used data to what is necessary for a baseline, to reduce file size. + foreach (var usage in used.Usages) + { + usage.AssetsFile = null; + } + used.Usages = used.Usages.Distinct().ToArray(); + + Directory.CreateDirectory(Path.GetDirectoryName(OutputBaselineFile)); + File.WriteAllText(OutputBaselineFile, used.ToXml().ToString()); + + Directory.CreateDirectory(Path.GetDirectoryName(OutputReportFile)); + File.WriteAllText(OutputReportFile, report.ToString()); + + if (tellUserToUpdateBaseline) + { + Log.LogWarning( + "Prebuilt usages are different from the baseline. If detected changes are " + + "acceptable, update baseline with:\n" + + $"cp '{OutputBaselineFile}' '{BaselineDataFile}'"); + } + + return !Log.HasLoggedErrors; + } + + private static Comparison Compare(IEnumerable actual, IEnumerable baseline) + { + return new Comparison(actual.ToArray(), baseline.ToArray()); + } + + private class Comparison + { + public T[] Added { get; } + public T[] Removed { get; } + public T[] Unchanged { get; } + + public Comparison( + IEnumerable actual, + IEnumerable baseline) + { + Added = actual.Except(baseline).ToArray(); + Removed = baseline.Except(actual).ToArray(); + Unchanged = actual.Intersect(baseline).ToArray(); + } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WritePackageUsageData.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WritePackageUsageData.cs new file mode 100644 index 000000000..73c7026a5 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WritePackageUsageData.cs @@ -0,0 +1,284 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NuGet.Packaging.Core; +using NuGet.Versioning; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; +using Task = Microsoft.Build.Utilities.Task; + +namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport +{ + public class WritePackageUsageData : Task + { + public string[] RestoredPackageFiles { get; set; } + public string[] TarballPrebuiltPackageFiles { get; set; } + public string[] ReferencePackageFiles { get; set; } + public string[] SourceBuiltPackageFiles { get; set; } + + /// + /// Specific PackageInfo items to check for usage. An alternative to passing lists of nupkgs + /// when the nupkgs have already been parsed to get package info items. + /// + /// %(Identity): Path to the original nupkg. + /// %(PackageId): Identity of the package. + /// %(PackageVersion): Version of the package. + /// + public ITaskItem[] NuGetPackageInfos { get; set; } + + /// + /// runtime.json files (from Microsoft.NETCore.Platforms) to use to look for the set of all + /// possible runtimes. This is used to determine which part of a package id is its + /// 'runtime.{rid}.' prefix, if it has the prefix. + /// + public string[] PlatformsRuntimeJsonFiles { get; set; } + + /// + /// Keep track of the RID built that caused these usages. + /// + public string TargetRid { get; set; } + + /// + /// Project directories to scan for project.assets.json files. If these directories contain + /// one another, the project.assets.json files is reported as belonging to the first project + /// directory that contains it. For useful results, put the leafmost directories first. + /// + /// This isn't used here, but it's included in the usage data so report generation can + /// happen independently of commits that add/remove submodules. + /// + public string[] ProjectDirectories { get; set; } + + /// + /// A root dir that contains all ProjectDirectories. This is used to find the relative path + /// of each usage. + /// + [Required] + public string RootDir { get; set; } + + /// + /// project.assets.json files to ignore, for example, because they are checked-in assets not + /// generated during source-build and cause false positives. + /// + public string[] IgnoredProjectAssetsJsonFiles { get; set; } + + /// + /// Output usage data JSON file path. + /// + [Required] + public string DataFile { get; set; } + + /// + /// If passed, the path of the archive file to generate that includes a copy of all + /// project.asset.json files found. + /// + public string ProjectAssetsJsonArchiveFile { get; set; } + + public override bool Execute() + { + DateTime startTime = DateTime.Now; + Log.LogMessage(MessageImportance.High, "Writing package usage data..."); + + string[] projectDirectoriesOutsideRoot = ProjectDirectories.NullAsEmpty() + .Where(dir => !dir.StartsWith(RootDir, StringComparison.Ordinal)) + .ToArray(); + + if (projectDirectoriesOutsideRoot.Any()) + { + throw new ArgumentException( + $"All ProjectDirectories must be in RootDir '{RootDir}', but found " + + string.Join(", ", projectDirectoriesOutsideRoot)); + } + + Log.LogMessage(MessageImportance.Low, "Finding set of RIDs..."); + + string[] possibleRids = PlatformsRuntimeJsonFiles.NullAsEmpty() + .SelectMany(ReadRidsFromRuntimeJson) + .Distinct() + .ToArray(); + + Log.LogMessage(MessageImportance.Low, "Reading package identities..."); + + PackageIdentity[] restored = RestoredPackageFiles.NullAsEmpty() + .Select(ReadNuGetPackageInfos.ReadIdentity) + .Distinct() + .ToArray(); + + PackageIdentity[] tarballPrebuilt = TarballPrebuiltPackageFiles.NullAsEmpty() + .Select(ReadNuGetPackageInfos.ReadIdentity) + .Distinct() + .ToArray(); + + PackageIdentity[] referencePackages = ReferencePackageFiles.NullAsEmpty() + .Select(ReadNuGetPackageInfos.ReadIdentity) + .Distinct() + .ToArray(); + + PackageIdentity[] sourceBuilt = SourceBuiltPackageFiles.NullAsEmpty() + .Select(ReadNuGetPackageInfos.ReadIdentity) + .Distinct() + .ToArray(); + + IEnumerable prebuilt = restored.Except(sourceBuilt).Except(referencePackages); + + PackageIdentity[] toCheck = NuGetPackageInfos.NullAsEmpty() + .Select(item => new PackageIdentity( + item.GetMetadata("PackageId"), + NuGetVersion.Parse(item.GetMetadata("PackageVersion")))) + .Concat(prebuilt) + .ToArray(); + + Log.LogMessage(MessageImportance.Low, "Finding project.assets.json files..."); + + string[] assetFiles = Directory + .GetFiles(RootDir, "project.assets.json", SearchOption.AllDirectories) + .Select(GetPathRelativeToRoot) + .Except(IgnoredProjectAssetsJsonFiles.NullAsEmpty().Select(GetPathRelativeToRoot)) + .ToArray(); + + if (!string.IsNullOrEmpty(ProjectAssetsJsonArchiveFile)) + { + Log.LogMessage(MessageImportance.Low, "Archiving project.assets.json files..."); + + Directory.CreateDirectory(Path.GetDirectoryName(ProjectAssetsJsonArchiveFile)); + + using (var projectAssetArchive = new ZipArchive( + File.Open( + ProjectAssetsJsonArchiveFile, + FileMode.Create, + FileAccess.ReadWrite), + ZipArchiveMode.Create)) + { + // Only one entry can be open at a time, so don't do this during the Parallel + // ForEach later. + foreach (var relativePath in assetFiles) + { + using (var stream = File.OpenRead(Path.Combine(RootDir, relativePath))) + using (Stream entryWriter = projectAssetArchive + .CreateEntry(relativePath, CompressionLevel.Optimal) + .Open()) + { + stream.CopyTo(entryWriter); + } + } + } + } + + Log.LogMessage(MessageImportance.Low, "Reading usage info..."); + + var usages = new ConcurrentBag(); + + Parallel.ForEach( + assetFiles, + assetFile => + { + JObject jObj; + + using (var file = File.OpenRead(Path.Combine(RootDir, assetFile))) + using (var reader = new StreamReader(file)) + using (var jsonReader = new JsonTextReader(reader)) + { + jObj = (JObject)JToken.ReadFrom(jsonReader); + } + + var properties = new HashSet( + jObj.SelectTokens("$.targets.*").Children() + .Concat(jObj.SelectToken("$.libraries")) + .Select(t => ((JProperty)t).Name) + .Distinct(), + StringComparer.OrdinalIgnoreCase); + + var directDependencies = jObj.SelectTokens("$.project.frameworks.*.dependencies").Children().Select(dep => + new + { + name = ((JProperty)dep).Name, + target = dep.SelectToken("$..target")?.ToString(), + version = VersionRange.Parse(dep.SelectToken("$..version")?.ToString()), + autoReferenced = dep.SelectToken("$..autoReferenced")?.ToString() == "True", + }) + .ToArray(); + + foreach (var identity in toCheck + .Where(id => properties.Contains(id.Id + "/" + id.Version.OriginalVersion))) + { + var directDependency = + directDependencies?.FirstOrDefault( + d => d.name == identity.Id && + d.version.Satisfies(identity.Version)); + usages.Add(Usage.Create( + assetFile, + identity, + directDependency != null, + directDependency?.autoReferenced == true, + possibleRids)); + } + }); + + Log.LogMessage(MessageImportance.Low, "Searching for unused packages..."); + + foreach (PackageIdentity restoredWithoutUsagesFound in + toCheck.Except(usages.Select(u => u.PackageIdentity))) + { + usages.Add(Usage.Create( + null, + restoredWithoutUsagesFound, + false, + false, + possibleRids)); + } + + // Packages that were included in the tarball as prebuilts, but weren't even restored. + PackageIdentity[] neverRestoredTarballPrebuilts = tarballPrebuilt + .Except(restored) + .ToArray(); + + Log.LogMessage(MessageImportance.Low, $"Writing data to '{DataFile}'..."); + + var data = new UsageData + { + CreatedByRid = TargetRid, + Usages = usages.ToArray(), + NeverRestoredTarballPrebuilts = neverRestoredTarballPrebuilts, + ProjectDirectories = ProjectDirectories + ?.Select(GetPathRelativeToRoot) + .ToArray() + }; + + Directory.CreateDirectory(Path.GetDirectoryName(DataFile)); + File.WriteAllText(DataFile, data.ToXml().ToString()); + + Log.LogMessage( + MessageImportance.High, + $"Writing package usage data... done. Took {DateTime.Now - startTime}"); + + return !Log.HasLoggedErrors; + } + + private string GetPathRelativeToRoot(string path) + { + if (path.StartsWith(RootDir)) + { + return path.Substring(RootDir.Length).Replace(Path.DirectorySeparatorChar, '/'); + } + + throw new ArgumentException($"Path '{path}' is not within RootDir '{RootDir}'"); + } + + private static string[] ReadRidsFromRuntimeJson(string path) + { + var root = JObject.Parse(File.ReadAllText(path)); + return root["runtimes"] + .Values() + .Select(o => o.Name) + .ToArray(); + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WriteUsageBurndownData.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WriteUsageBurndownData.cs new file mode 100644 index 000000000..ab40fa590 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WriteUsageBurndownData.cs @@ -0,0 +1,130 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NuGet.Packaging.Core; +using NuGet.Versioning; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using System.Xml.Linq; +using Task = Microsoft.Build.Utilities.Task; + +namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport +{ + public class WriteUsageBurndownData : Task + { + /// + /// Specifies the root directory for git. + /// Note: Requires a trailing "/" when specifying the directory. + /// + [Required] + public string RootDirectory { get; set; } + + /// + /// Specifies the path to the prebuilt baseline file + /// to be used to generate the burndown. + /// + [Required] + public string PrebuiltBaselineFile { get; set; } + + /// + /// Output data CSV file. + /// + [Required] + public string OutputFilePath { get; set; } + + public override bool Execute() + { + string baselineRelativeFileName = PrebuiltBaselineFile.Replace(RootDirectory, ""); + string gitLogCommand = $"log --first-parent --pretty=format:%H,%f,%ci -- {PrebuiltBaselineFile}"; + + DateTime startTime = DateTime.Now; + Log.LogMessage(MessageImportance.High, "Generating summary usage burndown data..."); + + ParallelQuery data = ExecuteGitCommand(RootDirectory, gitLogCommand).AsParallel().Select(commitLine => + { + var splitLine = commitLine.Split(','); + var commit = new Commit() + { + Sha = splitLine[0], + Title = splitLine[1], + CommitDate = DateTime.Parse(splitLine[2]) + }; + string fileContents = GetFileContents(baselineRelativeFileName, commit.Sha); + Usage[] usages = UsageData.Parse(XElement.Parse(fileContents)).Usages.NullAsEmpty().ToArray(); + commit.PackageVersionCount = usages.Count(); + commit.PackageCount = usages.GroupBy(i => i.PackageIdentity.Id).Select(grp => grp.First()).Count(); + return commit; + }) + .Select(c => c.ToString()); + + Directory.CreateDirectory(Path.GetDirectoryName(OutputFilePath)); + + File.WriteAllLines(OutputFilePath, data); + + Log.LogMessage( + MessageImportance.High, + $"Generating summary usage burndown data at {OutputFilePath} done. Took {DateTime.Now - startTime}"); + + return !Log.HasLoggedErrors; + } + + /// + /// Get the contents of a git file based on the commit sha. + /// + /// The relative path (from the git root) to the file. + /// The commit sha for the version of the file to get. + /// The contents of the specified file. + private string GetFileContents(string relativeFilePath, string commitSha) + { + WebClient client = new WebClient(); + var xmlString = client.DownloadString($"https://raw.githubusercontent.com/dotnet/source-build/{commitSha}/{relativeFilePath.Replace('\\', '/')}"); + return xmlString; + } + + /// + /// Executes a git command and returns the result. + /// + /// The working directory for the git command. + /// The git command to execute. + /// An array of the output lines of the git command. + private string[] ExecuteGitCommand(string workingDirectory, string command) + { + string[] returnData; + Process _process = new Process(); + _process.StartInfo.FileName = "git"; + _process.StartInfo.Arguments = command; + _process.StartInfo.WorkingDirectory = workingDirectory; + _process.StartInfo.RedirectStandardOutput = true; + _process.StartInfo.UseShellExecute = false; + _process.Start(); + returnData = _process.StandardOutput.ReadToEnd().Split('\n'); + _process.WaitForExit(); + return returnData; + } + + private class Commit + { + public string Sha { get; set; } + public string Title { get; set; } + public DateTime CommitDate { get; set; } + public int PackageVersionCount { get; set; } + public int PackageCount { get; set; } + + public override string ToString() + { + return $"{Sha}, {Title}, {CommitDate}, {PackageVersionCount}, {PackageCount}"; + } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WriteUsageReports.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WriteUsageReports.cs new file mode 100644 index 000000000..e236d16a3 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WriteUsageReports.cs @@ -0,0 +1,303 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Microsoft.DotNet.Build.Tasks; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport +{ + public class WriteUsageReports : Task + { + private const string SnapshotPrefix = "PackageVersions.props.pre."; + private const string SnapshotSuffix = ".xml"; + + /// + /// Source usage data JSON file. + /// + [Required] + public string DataFile { get; set; } + + /// + /// A set of "PackageVersions.props.pre.{repo}.xml" files. They are analyzed to find + /// packages built during source-build, and which repo built them. This info is added to the + /// report. New packages are associated to a repo by going through each PVP in ascending + /// file modification order. + /// + public ITaskItem[] PackageVersionPropsSnapshots { get; set; } + + /// + /// Infos that associate packages to the ProdCon build they're from. + /// + /// %(PackageId): Identity of the package. + /// %(OriginBuildName): Name of the build that produced this package. + /// + public ITaskItem[] ProdConPackageInfos { get; set; } + + /// + /// Path to a ProdCon build manifest file (build.xml) as an alternative way to pass + /// ProdConPackageInfos items. + /// + public string ProdConBuildManifestFile { get; set; } + + /// + /// File containing the results of poisoning the prebuilts. Example format: + /// + /// MATCH: output built\dotnet-sdk-...\System.Collections.dll(hash 4b...31) matches one of: + /// intermediate\netstandard.library.2.0.1.nupkg\build\...\System.Collections.dll + /// + /// The usage report reads this file, looking for 'intermediate\*.nupkg' to annotate. + /// + public string PoisonedReportFile { get; set; } + + [Required] + public string OutputDirectory { get; set; } + + public override bool Execute() + { + UsageData data = UsageData.Parse(XElement.Parse(File.ReadAllText(DataFile))); + + IEnumerable sourceBuildRepoOutputs = GetSourceBuildRepoOutputs(); + + // Map package id to the build name that created them in a ProdCon build. + var prodConPackageOrigin = new Dictionary( + StringComparer.OrdinalIgnoreCase); + + foreach (ITaskItem item in ProdConPackageInfos.NullAsEmpty()) + { + AddProdConPackage( + prodConPackageOrigin, + item.GetMetadata("PackageId"), + item.GetMetadata("OriginBuildName")); + } + + if (File.Exists(ProdConBuildManifestFile)) + { + var xml = XElement.Parse(File.ReadAllText(ProdConBuildManifestFile)); + foreach (var x in xml.Descendants("Package")) + { + AddProdConPackage( + prodConPackageOrigin, + x.Attribute("Id")?.Value, + x.Attribute("OriginBuildName")?.Value); + } + } + + var poisonNupkgFilenames = new HashSet(StringComparer.OrdinalIgnoreCase); + + if (File.Exists(PoisonedReportFile)) + { + foreach (string line in File.ReadAllLines(PoisonedReportFile)) + { + string[] segments = line.Split('\\'); + if (segments.Length > 2 && + segments[0].Trim() == "intermediate" && + segments[1].EndsWith(".nupkg")) + { + poisonNupkgFilenames.Add(Path.GetFileNameWithoutExtension(segments[1])); + } + } + } + + var report = new XElement("AnnotatedUsages"); + + var annotatedUsages = data.Usages.NullAsEmpty() + .Select(usage => + { + string id = usage.PackageIdentity.Id; + string version = usage.PackageIdentity.Version.OriginalVersion; + + string pvpIdent = WriteBuildOutputProps.GetPropertyName(id); + + var sourceBuildCreator = new StringBuilder(); + foreach (RepoOutput output in sourceBuildRepoOutputs) + { + foreach (PackageVersionPropsElement p in output.Built) + { + if (p.Name.Equals(pvpIdent, StringComparison.OrdinalIgnoreCase)) + { + if (sourceBuildCreator.Length != 0) + { + sourceBuildCreator.Append(" "); + } + sourceBuildCreator.Append(output.Repo); + sourceBuildCreator.Append(" "); + sourceBuildCreator.Append(p.Name); + sourceBuildCreator.Append("/"); + sourceBuildCreator.Append(p.Version); + } + } + } + + prodConPackageOrigin.TryGetValue(id, out string prodConCreator); + + return new AnnotatedUsage + { + Usage = usage, + + Project = data.ProjectDirectories + ?.FirstOrDefault(p => usage.AssetsFile?.StartsWith(p) ?? false), + + SourceBuildPackageIdCreator = sourceBuildCreator.Length == 0 + ? null + : sourceBuildCreator.ToString(), + + ProdConPackageIdCreator = prodConCreator, + + TestProjectByHeuristic = IsTestUsageByHeuristic(usage), + + EndsUpInOutput = poisonNupkgFilenames.Contains($"{id}.{version}") + }; + }) + .ToArray(); + + foreach (var onlyTestProjectUsage in annotatedUsages + .GroupBy(u => u.Usage.PackageIdentity) + .Where(g => g.All(u => u.TestProjectByHeuristic)) + .SelectMany(g => g)) + { + onlyTestProjectUsage.TestProjectOnlyByHeuristic = true; + } + + report.Add(annotatedUsages.Select(u => u.ToXml())); + + Directory.CreateDirectory(OutputDirectory); + + File.WriteAllText( + Path.Combine(OutputDirectory, "annotated-usage.xml"), + report.ToString()); + + return !Log.HasLoggedErrors; + } + + private RepoOutput[] GetSourceBuildRepoOutputs() + { + var pvpSnapshotFiles = PackageVersionPropsSnapshots.NullAsEmpty() + .Select(item => + { + var content = File.ReadAllText(item.ItemSpec); + return new + { + Path = item.ItemSpec, + Content = content, + Xml = XElement.Parse(content) + }; + }) + .OrderBy(snapshot => + { + // Get the embedded creation time if possible: the file's original metadata may + // have been destroyed by copying, zipping, etc. + string creationTime = snapshot.Xml + // Get the second PropertyGroup. + .Elements().Skip(1).FirstOrDefault() + // Get the creation time element. + ?.Element(snapshot.Xml + .GetDefaultNamespace() + .GetName(WriteBuildOutputProps.CreationTimePropertyName)) + ?.Value; + + if (string.IsNullOrEmpty(creationTime)) + { + Log.LogError($"No creation time property found in snapshot {snapshot.Path}"); + return default(DateTime); + } + + return new DateTime(long.Parse(creationTime)); + }) + .Select(snapshot => + { + string filename = Path.GetFileName(snapshot.Path); + return new + { + Repo = filename.Substring( + SnapshotPrefix.Length, + filename.Length - SnapshotPrefix.Length - SnapshotSuffix.Length), + PackageVersionProp = PackageVersionPropsElement.Parse(snapshot.Xml) + }; + }) + .ToArray(); + + return pvpSnapshotFiles.Skip(1) + .Zip(pvpSnapshotFiles, (pvp, prev) => new RepoOutput + { + Repo = prev.Repo, + Built = pvp.PackageVersionProp.Except(prev.PackageVersionProp).ToArray() + }) + .ToArray(); + } + + private void AddProdConPackage( + Dictionary packageOrigin, + string id, + string origin) + { + if (!string.IsNullOrEmpty(id) && + !string.IsNullOrEmpty(origin)) + { + packageOrigin[id] = origin; + } + } + + public static bool IsTestUsageByHeuristic(Usage usage) + { + string[] assetsFileParts = usage.AssetsFile?.Split('/', '\\'); + + // If the dir name ends in Test(s), it's probably a test project. + // Ignore the first two segments to avoid classifying everything in "src/vstest". + // This also catches "test" dirs that contain many test projects. + if (assetsFileParts?.Skip(2).Any(p => + p.EndsWith("Tests", StringComparison.OrdinalIgnoreCase) || + p.EndsWith("Test", StringComparison.OrdinalIgnoreCase)) == true) + { + return true; + } + + // CoreFX restores test dependencies during this sync project. + if (assetsFileParts?.Contains("XUnit.Runtime") == true) + { + return true; + } + + return false; + } + + private class RepoOutput + { + public string Repo { get; set; } + public PackageVersionPropsElement[] Built { get; set; } + } + + private struct PackageVersionPropsElement + { + public static PackageVersionPropsElement[] Parse(XElement xml) + { + return xml + // Get the single PropertyGroup + .Elements() + .First() + // Get all *PackageVersion property elements. + .Elements() + .Select(x => new PackageVersionPropsElement( + x.Name.LocalName, + x.Nodes().OfType().First().Value)) + .ToArray(); + } + + public string Name { get; } + public string Version { get; } + + public PackageVersionPropsElement(string name, string version) + { + Name = name; + Version = version; + } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/XmlParsingHelpers.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/XmlParsingHelpers.cs new file mode 100644 index 000000000..c30173fff --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/XmlParsingHelpers.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using NuGet.Packaging.Core; +using NuGet.Versioning; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport +{ + internal static class XmlParsingHelpers + { + public static XElement ToXElement(this PackageIdentity ident) => new XElement( + "PackageIdentity", + new XAttribute("Id", ident.Id), + new XAttribute("Version", ident.Version.OriginalVersion)); + + public static XAttribute ToXAttributeIfNotNull(this object value, string name) => + value == null ? null : new XAttribute(name, value); + + public static XAttribute ToXAttributeIfTrue(this bool value, string name) => + value == false ? null : new XAttribute(name, value); + + public static PackageIdentity ParsePackageIdentity(XElement xml) => new PackageIdentity( + xml.Attribute("Id").Value, + NuGetVersion.Parse(xml.Attribute("Version").Value)); + + public static IOrderedEnumerable OrderByOrdinal( + this IEnumerable source, + Func selector) + { + return source.OrderBy(selector, StringComparer.Ordinal); + } + + public static IOrderedEnumerable ThenByOrdinal( + this IOrderedEnumerable source, + Func selector) + { + return source.ThenBy(selector, StringComparer.Ordinal); + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteBuildOutputProps.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteBuildOutputProps.cs new file mode 100644 index 000000000..fb1d2fe4d --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteBuildOutputProps.cs @@ -0,0 +1,142 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using NuGet.Versioning; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class WriteBuildOutputProps : Task + { + private static readonly Regex InvalidElementNameCharRegex = new Regex(@"(^|[^A-Za-z0-9])(?.)"); + + public const string CreationTimePropertyName = "BuildOutputPropsCreationTime"; + + [Required] + public ITaskItem[] NuGetPackages { get; set; } + + [Required] + public string OutputPath { get; set; } + + /// + /// Adds a second PropertyGroup to the output XML containing a property with the time of + /// creation in UTC DateTime Ticks. This can be used to track creation time in situations + /// where file metadata isn't reliable or preserved. + /// + public bool IncludeCreationTimeProperty { get; set; } + + /// + /// Properties to add to the build output props, which may not exist as nupkgs. + /// FOr example, this is used to pass the version of the CLI toolset archives. + /// + /// %(Identity): Package identity. + /// %(Version): Package version. + /// + public ITaskItem[] ExtraProperties { get; set; } + + /// + /// Additional assets to be added to the build output props. + /// i.e. /bin/obj/x64/Release/blobs/Toolset/3.0.100 + /// This parameter is the / portion only, and the asset + /// must be in a / folder. + /// + public string[] AdditionalAssetDirs { get; set; } + + public override bool Execute() + { + PackageIdentity[] latestPackages = NuGetPackages + .Select(item => + { + using (var reader = new PackageArchiveReader(item.GetMetadata("FullPath"))) + { + return reader.GetIdentity(); + } + }) + .GroupBy(identity => identity.Id) + .Select(g => g.OrderBy(id => id.Version).Last()) + .OrderBy(id => id.Id) + .ToArray(); + + var additionalAssets = (AdditionalAssetDirs ?? new string[0]) + .Where(Directory.Exists) + .Where(dir => Directory.GetDirectories(dir).Count() > 0) + .Select(dir => new { + Name = new DirectoryInfo(dir).Name + "Version", + Version = new DirectoryInfo(Directory.EnumerateDirectories(dir).OrderBy(s => s).Last()).Name + }).ToArray(); + + Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); + + using (var outStream = File.Open(OutputPath, FileMode.Create)) + using (var sw = new StreamWriter(outStream, new UTF8Encoding(false))) + { + sw.WriteLine(@""); + sw.WriteLine(@""); + sw.WriteLine(@" "); + foreach (PackageIdentity packageIdentity in latestPackages) + { + string propertyName = GetPropertyName(packageIdentity.Id); + sw.WriteLine($" <{propertyName}>{packageIdentity.Version}"); + + propertyName = GetAlternatePropertyName(packageIdentity.Id); + sw.WriteLine($" <{propertyName}>{packageIdentity.Version}"); + } + foreach (var extraProp in ExtraProperties ?? Enumerable.Empty()) + { + string propertyName = extraProp.GetMetadata("Identity"); + bool doNotOverwrite = false; + string overwriteCondition = string.Empty; + if (bool.TryParse(extraProp.GetMetadata("DoNotOverwrite"), out doNotOverwrite) && doNotOverwrite) + { + overwriteCondition = $" Condition=\"'$({propertyName})' == ''\""; + } + sw.WriteLine($" <{propertyName}{overwriteCondition}>{extraProp.GetMetadata("Version")}"); + } + foreach (var additionalAsset in additionalAssets) + { + sw.WriteLine($" <{additionalAsset.Name}>{additionalAsset.Version}"); + } + sw.WriteLine(@" "); + if (IncludeCreationTimeProperty) + { + sw.WriteLine(@" "); + sw.WriteLine($@" <{CreationTimePropertyName}>{DateTime.UtcNow.Ticks}"); + sw.WriteLine(@" "); + } + sw.WriteLine(@""); + } + + return true; + } + + public static string GetPropertyName(string id) + { + string formattedId = InvalidElementNameCharRegex.Replace( + id, + match => match.Groups?["FirstPartChar"].Value.ToUpperInvariant() + ?? string.Empty); + + return $"{formattedId}PackageVersion"; + } + + public static string GetAlternatePropertyName(string id) + { + string formattedId = InvalidElementNameCharRegex.Replace( + id, + match => match.Groups?["FirstPartChar"].Value.ToUpperInvariant() + ?? string.Empty); + + return $"{formattedId}Version"; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceAndVersionProps.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceAndVersionProps.cs new file mode 100644 index 000000000..ee8a9ba7d --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceAndVersionProps.cs @@ -0,0 +1,156 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using NuGet.Versioning; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class WriteRestoreSourceAndVersionProps : Task + { + private static readonly Regex InvalidElementNameCharRegex = new Regex(@"(^|[^A-Za-z0-9])(?.)"); + + public const string CreationTimePropertyName = "BuildOutputPropsCreationTime"; + + [Required] + public ITaskItem[] RestoreSources { get; set; } + + [Required] + public ITaskItem[] NuGetPackages { get; set; } + + [Required] + public string OutputPath { get; set; } + + /// + /// Adds a second PropertyGroup to the output XML containing a property with the time of + /// creation in UTC DateTime Ticks. This can be used to track creation time in situations + /// where file metadata isn't reliable or preserved. + /// + public bool IncludeCreationTimeProperty { get; set; } + + /// + /// Properties to add to the build output props, which may not exist as nupkgs. + /// FOr example, this is used to pass the version of the CLI toolset archives. + /// + /// %(Identity): Package identity. + /// %(Version): Package version. + /// + public ITaskItem[] ExtraProperties { get; set; } + + /// + /// Additional assets to be added to the build output props. + /// i.e. /bin/obj/x64/Release/blobs/Toolset/3.0.100 + /// This parameter is the / portion only, and the asset + /// must be in a / folder. + /// + public string[] AdditionalAssetDirs { get; set; } + + public override bool Execute() + { + PackageIdentity[] latestPackages = NuGetPackages + .Select(item => + { + using (var reader = new PackageArchiveReader(item.GetMetadata("FullPath"))) + { + return reader.GetIdentity(); + } + }) + .GroupBy(identity => identity.Id) + .Select(g => g.OrderBy(id => id.Version).Last()) + .OrderBy(id => id.Id) + .ToArray(); + + var additionalAssets = (AdditionalAssetDirs ?? new string[0]) + .Where(Directory.Exists) + .Where(dir => Directory.GetDirectories(dir).Count() > 0) + .Select(dir => new { + Name = new DirectoryInfo(dir).Name + "Version", + Version = new DirectoryInfo(Directory.EnumerateDirectories(dir).OrderBy(s => s).Last()).Name + }).ToArray(); + + Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); + + using (var outStream = File.Open(OutputPath, FileMode.Create)) + using (var sw = new StreamWriter(outStream, new UTF8Encoding(false))) + { + sw.WriteLine(@""); + sw.WriteLine(@""); + sw.WriteLine(@" "); + sw.WriteLine(@" "); + foreach (ITaskItem restoreSourceItem in RestoreSources) + { + sw.WriteLine($" {restoreSourceItem.ItemSpec};"); + } + sw.WriteLine(@" "); + sw.WriteLine(@" "); + + sw.WriteLine(@" "); + foreach (PackageIdentity packageIdentity in latestPackages) + { + string propertyName = GetPropertyName(packageIdentity.Id); + string shortPropertyName = GetShortPropertyName(packageIdentity.Id); + + sw.WriteLine($" <{propertyName}>{packageIdentity.Version}"); + sw.WriteLine($" <{shortPropertyName}>{packageIdentity.Version}"); + } + foreach (var extraProp in ExtraProperties ?? Enumerable.Empty()) + { + string propertyName = extraProp.GetMetadata("Identity"); + bool doNotOverwrite = false; + string overwriteCondition = string.Empty; + if (bool.TryParse(extraProp.GetMetadata("DoNotOverwrite"), out doNotOverwrite) && doNotOverwrite) + { + overwriteCondition = $" Condition=\"'$({propertyName})' == ''\""; + } + sw.WriteLine($" <{propertyName}{overwriteCondition}>{extraProp.GetMetadata("Version")}"); + } + foreach (var additionalAsset in additionalAssets) + { + sw.WriteLine($" <{additionalAsset.Name}>{additionalAsset.Version}"); + } + sw.WriteLine(@" "); + + if (IncludeCreationTimeProperty) + { + sw.WriteLine(@" "); + sw.WriteLine($@" <{CreationTimePropertyName}>{DateTime.UtcNow.Ticks}"); + sw.WriteLine(@" "); + } + + sw.WriteLine(@""); + } + + return true; + } + + public static string GetPropertyName(string id) + { + string formattedId = InvalidElementNameCharRegex.Replace( + id, + match => match.Groups?["FirstPartChar"].Value.ToUpperInvariant() + ?? string.Empty); + + return $"{formattedId}PackageVersion"; + } + + public static string GetShortPropertyName(string id) + { + string formattedId = InvalidElementNameCharRegex.Replace( + id, + match => match.Groups?["FirstPartChar"].Value.ToUpperInvariant() + ?? string.Empty); + + return $"{formattedId}Version"; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceProps.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceProps.cs new file mode 100644 index 000000000..8e983531b --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceProps.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System.IO; +using System.Text; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class WriteRestoreSourceProps : Task + { + [Required] + public ITaskItem[] RestoreSources { get; set; } + + [Required] + public string OutputPath { get; set; } + + public override bool Execute() + { + Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); + + using (var outStream = File.Open(OutputPath, FileMode.Create)) + using (var sw = new StreamWriter(outStream, new UTF8Encoding(false))) + { + sw.WriteLine(@""); + sw.WriteLine(@""); + sw.WriteLine(@" "); + sw.WriteLine(@" "); + foreach (ITaskItem restoreSourceItem in RestoreSources) + { + sw.WriteLine($" {restoreSourceItem.ItemSpec};"); + } + sw.WriteLine(@" "); + sw.WriteLine(@" "); + sw.WriteLine(@""); + } + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteSourceRepoProperties.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteSourceRepoProperties.cs new file mode 100644 index 000000000..5b96f9f87 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteSourceRepoProperties.cs @@ -0,0 +1,317 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Tasks; +using Microsoft.Build.Utilities; +using Microsoft.DotNet.SourceBuild.Tasks.Models; +using NuGet.Versioning; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml.Serialization; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class WriteSourceRepoProperties : Task + { + [Required] + public string VersionDetailsFile { get; set; } + + [Required] + public string ClonedSubmoduleGitRootDirectory { get; set; } + + [Required] + public string ClonedSubmoduleDirectory { get; set; } + + [Required] + public string SourceBuildMetadataDir { get; set; } + + public override bool Execute() + { + var serializer = new XmlSerializer(typeof(VersionDetails)); + + VersionDetails versionDetails = null; + using (var stream = File.OpenRead(VersionDetailsFile)) + { + versionDetails = (VersionDetails)serializer.Deserialize(stream); + } + + var allRepoProps = new Dictionary(); + + foreach (var dep in versionDetails.ToolsetDependencies.Concat(versionDetails.ProductDependencies)) + { + Log.LogMessage(MessageImportance.Normal, $"[{DateTimeOffset.Now}] Starting dependency {dep.ToString()}"); + string repoPath = DeriveRepoPath(ClonedSubmoduleDirectory, dep.Uri, dep.Sha); + string repoGitDir = DeriveRepoGitDirPath(ClonedSubmoduleGitRootDirectory, dep.Uri); + if (Directory.Exists(repoGitDir)) + { + foreach (string repoName in GetRepoNamesOrDefault(dep)) + { + string safeRepoName = repoName.Replace("-", ""); + try + { + WriteMinimalMetadata(repoPath, dep.Uri, dep.Sha); + WriteSourceBuildMetadata(SourceBuildMetadataDir, repoGitDir, dep); + if (File.Exists(Path.Combine(repoPath, ".gitmodules"))) + { + HandleSubmodules(repoPath, repoGitDir, dep); + } + allRepoProps[$"{safeRepoName}GitCommitHash"] = dep.Sha; + allRepoProps[$"{safeRepoName}OutputPackageVersion"] = dep.Version; + } + catch (Exception e) + { + Log.LogErrorFromException(e, true, true, null); + } + } + } + else + { + Log.LogMessage(MessageImportance.Normal, $"[{DateTimeOffset.Now}] Skipping dependency {dep.ToString()} - git dir {repoGitDir} doesn't exist"); + } + } + string allRepoPropsPath = Path.Combine(SourceBuildMetadataDir, "AllRepoVersions.props"); + Log.LogMessage(MessageImportance.Normal, $"[{DateTimeOffset.Now}] Writing all repo versions to {allRepoPropsPath}"); + WritePropsFile(allRepoPropsPath, allRepoProps); + + return !Log.HasLoggedErrors; + } + + private void WriteSourceBuildMetadata(string sourceBuildMetadataPath, string repoGitDir, Dependency dependency) + { + foreach (string repoName in GetRepoNamesOrDefault(dependency)) + { + string propsPath = Path.Combine(sourceBuildMetadataPath, $"{repoName}.props"); + string commitCount = GetCommitCount(repoGitDir, dependency.Sha); + DerivedVersion derivedVersion = GetVersionInfo(dependency.Version, commitCount); + var repoProps = new Dictionary + { + ["GitCommitHash"] = dependency.Sha, + ["GitCommitCount"] = commitCount, + ["GitCommitDate"] = GetCommitDate(repoGitDir, dependency.Sha), + ["OfficialBuildId"] = derivedVersion.OfficialBuildId, + ["OutputPackageVersion"] = dependency.Version, + ["PreReleaseVersionLabel"] = derivedVersion.PreReleaseVersionLabel, + ["IsStable"] = string.IsNullOrWhiteSpace(derivedVersion.PreReleaseVersionLabel) ? "true" : "false", + }; + WritePropsFile(propsPath, repoProps); + } + } + + + /// + /// Reverse a version in the Arcade style (https://github.com/dotnet/arcade/blob/fb92b14d8cd07cf44f8f7eefa8ac58d7ffd05f3f/src/Microsoft.DotNet.Arcade.Sdk/tools/Version.BeforeCommonTargets.targets#L18) + /// back to an OfficialBuildId + ReleaseLabel which we can then supply to get the same resulting version number. + /// + /// The complete version, e.g. 1.0.0-beta1-19720.5 + /// The current commit count of the repo. This is used for some repos that do not use the standard versioning scheme. + /// + private static DerivedVersion GetVersionInfo(string version, string commitCount) + { + var nugetVersion = new NuGetVersion(version); + + if (!string.IsNullOrWhiteSpace(nugetVersion.Release)) + { + var releaseParts = nugetVersion.Release.Split('-', '.'); + if (releaseParts.Length == 2) + { + if (releaseParts[1].TrimStart('0') == commitCount) + { + // core-sdk does this - OfficialBuildId is only used for their fake package and not in anything shipped + return new DerivedVersion { OfficialBuildId = DateTime.Now.ToString("yyyyMMdd.1"), PreReleaseVersionLabel = releaseParts[0] }; + } + else + { + // NuGet does this - arbitrary build IDs + return new DerivedVersion { OfficialBuildId = releaseParts[1], PreReleaseVersionLabel = releaseParts[0] }; + } + } + else if (releaseParts.Length == 3) + { + // VSTest uses full dates for the first part of their preview build numbers + if (DateTime.TryParseExact(releaseParts[1], "yyyyMMdd", new CultureInfo("en-US"), DateTimeStyles.AssumeLocal, out DateTime fullDate)) + { + return new DerivedVersion { OfficialBuildId = $"{releaseParts[1]}.{releaseParts[2]}", PreReleaseVersionLabel = releaseParts[0] }; + } + else if (int.TryParse(releaseParts[1], out int datePart) && int.TryParse(releaseParts[2], out int buildPart)) + { + if (datePart > 1 && datePart < 8 && buildPart > 1000 && buildPart < 10000) + { + return new DerivedVersion { OfficialBuildId = releaseParts[2], PreReleaseVersionLabel = $"{releaseParts[0]}.{releaseParts[1]}" }; + } + else + { + return new DerivedVersion { OfficialBuildId = $"20{((datePart / 1000))}{((datePart % 1000) / 50):D2}{(datePart % 50):D2}.{buildPart}", PreReleaseVersionLabel = releaseParts[0] }; + } + } + } + else if (releaseParts.Length == 4) + { + // new preview version style, e.g. 5.0.0-preview.7.20365.12 + if (int.TryParse(releaseParts[2], out int datePart) && int.TryParse(releaseParts[3], out int buildPart)) + { + return new DerivedVersion { OfficialBuildId = $"20{((datePart / 1000))}{((datePart % 1000) / 50):D2}{(datePart % 50):D2}.{buildPart}", PreReleaseVersionLabel = $"{releaseParts[0]}.{releaseParts[1]}" }; + } + } + } + else + { + // finalized version number (x.y.z) - probably not our code + // VSTest, Application Insights, Newtonsoft.Json do this + return new DerivedVersion { OfficialBuildId = DateTime.Now.ToString("yyyyMMdd.1"), PreReleaseVersionLabel = string.Empty }; + } + + throw new FormatException($"Can't derive a build ID from version {version} (commit count {commitCount}, release {string.Join(";", nugetVersion.Release.Split('-', '.'))})"); + } + + private static string GetDefaultRepoNameFromUrl(string repoUrl) + { + if (repoUrl.EndsWith(".git")) + { + repoUrl = repoUrl.Substring(0, repoUrl.Length - ".git".Length); + } + return repoUrl.Substring(repoUrl.LastIndexOf("/") + 1); + } + + private static IEnumerable GetRepoNamesOrDefault(Dependency dependency) + { + return dependency.RepoNames ?? new[] { GetDefaultRepoNameFromUrl(dependency.Uri) }; + } + + private static string DeriveRepoGitDirPath(string gitDirPath, string repoUrl) + { + return Path.Combine(gitDirPath, $"{GetDefaultRepoNameFromUrl(repoUrl)}.git"); + } + + private static string DeriveRepoPath(string sourceDirPath, string repoUrl, string hash) + { + // hash could actually be a branch or tag, make it filename-safe + hash = hash.Replace('/', '-').Replace('\\', '-').Replace('?', '-').Replace('*', '-').Replace(':', '-').Replace('|', '-').Replace('"', '-').Replace('<', '-').Replace('>', '-'); + return Path.Combine(sourceDirPath, $"{GetDefaultRepoNameFromUrl(repoUrl)}.{hash}"); + } + + private string GetCommitCount(string gitDir, string hash) + { + return RunGitCommand($"rev-list --count {hash}", gitDir: gitDir); + } + + private string GetCommitDate(string gitDir, string hash) + { + return RunGitCommand($"log -1 --format=%cd --date=short {hash}", gitDir: gitDir); + } + + private IEnumerable GetSubmoduleInfo(string gitModulesFilePath) + { + string submoduleProps = RunGitCommand($"config --file={gitModulesFilePath} --list"); + var submodulePathRegex = new Regex(@"submodule\.(?.*)\.path=(?.*)"); + foreach (Match m in submodulePathRegex.Matches(submoduleProps)) + { + yield return new SubmoduleInfo { Name = m.Groups["submoduleName"].Value, Path = m.Groups["submodulePath"].Value }; + } + } + + private string RunGitCommand(string command, string workTree = null, string gitDir = null) + { + // Windows Git requires these to be before the command + if (workTree != null) + { + command = $"--work-tree={workTree} {command}"; + } + if (gitDir != null) + { + command = $"--git-dir={gitDir} {command}"; + } + + var exec = new Exec + { + BuildEngine = BuildEngine, + Command = $"git {command}", + LogStandardErrorAsError = true, + ConsoleToMSBuild = true, + }; + + if (!exec.Execute() || exec.ExitCode != 0) + { + string error = string.Join(Environment.NewLine, exec.ConsoleOutput.Select(o => o.ItemSpec)); + throw new InvalidOperationException($"git command '{command}' failed with exit code {exec.ExitCode} and error {error ?? ""}"); + } + string output = string.Join(Environment.NewLine, exec.ConsoleOutput.Select(o => o.ItemSpec)); + return output.Trim(); + } + + private void HandleSubmodules(string sourceDirPath, string gitDirPath, Dependency dependency) + { + var gitModulesPath = Path.Combine(sourceDirPath, ".gitmodules"); + foreach (SubmoduleInfo submodule in GetSubmoduleInfo(gitModulesPath)) + { + WriteGitCommitMarkerFileForSubmodule(sourceDirPath, gitDirPath, dependency.Sha, submodule.Name, submodule.Path); + } + } + + private void WriteGitCommitMarkerFileForSubmodule(string sourceDirPath, string gitDirPath, string parentRepoSha, string submoduleName, string submodulePath) + { + var submoduleSha = GetSubmoduleCommit(gitDirPath, parentRepoSha, submodulePath); + var headDirectory = Path.Combine(sourceDirPath, submodulePath, ".git"); + var headPath = Path.Combine(headDirectory, "HEAD"); + Directory.CreateDirectory(headDirectory); + File.WriteAllText(headPath, submoduleSha); + } + + private static void WriteMinimalMetadata(string repoPath, string repoUrl, string hash) + { + var fakeGitDirPath = Path.Combine(repoPath, ".git"); + var fakeGitConfigPath = Path.Combine(fakeGitDirPath, "config"); + var fakeGitHeadPath = Path.Combine(fakeGitDirPath, "HEAD"); + + Directory.CreateDirectory(fakeGitDirPath); + File.WriteAllText(fakeGitHeadPath, hash); + File.WriteAllText(fakeGitConfigPath, $"[remote \"origin\"]{Environment.NewLine}url = \"{repoUrl}\""); + } + + + private string GetSubmoduleCommit(string gitDirPath, string parentRepoSha, string submodulePath) + { + var gitObjectList = RunGitCommand($"ls-tree {parentRepoSha} {submodulePath}", gitDir: gitDirPath); + var submoduleRegex = new Regex(@"\d{6}\s+commit\s+(?[a-fA-F0-9]{40})\s+(.+)"); + var submoduleMatch = submoduleRegex.Match(gitObjectList); + if (!submoduleMatch.Success) + { + throw new InvalidDataException($"Couldn't find a submodule commit in {gitObjectList} for {submodulePath}"); + } + return submoduleMatch.Groups["submoduleSha"].Value; + } + + private static void WritePropsFile(string filePath, Dictionary properties) + { + var content = new StringBuilder(); + content.AppendLine(""); + content.AppendLine(""); + content.AppendLine(" "); + foreach (var propName in properties.Keys.OrderBy(k => k)) + { + content.AppendLine($" <{propName}>{properties[propName]}"); + } + content.AppendLine(" "); + content.AppendLine(""); + File.WriteAllText(filePath, content.ToString()); + } + + private class DerivedVersion + { + internal string OfficialBuildId { get; set; } + internal string PreReleaseVersionLabel { get; set; } + } + + private class SubmoduleInfo + { + internal string Name { get; set; } + internal string Path { get; set; } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteVersionsFile.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteVersionsFile.cs new file mode 100755 index 000000000..e4ba279f7 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteVersionsFile.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Text; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Packaging; +using NuGet.Packaging.Core; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class WriteVersionsFile : Task + { + [Required] + public ITaskItem[] NugetPackages { get; set; } + + [Required] + public string OutputPath { get; set; } + + public override bool Execute() + { + Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); + + using (Stream outStream = File.Open(OutputPath, FileMode.Create)) + { + using (StreamWriter sw = new StreamWriter(outStream, new UTF8Encoding(false))) + { + foreach (ITaskItem nugetPackage in NugetPackages) + { + using (PackageArchiveReader par = new PackageArchiveReader(nugetPackage.GetMetadata("FullPath"))) + { + PackageIdentity packageIdentity = par.GetIdentity(); + sw.WriteLine($"{packageIdentity.Id} {packageIdentity.Version}"); + } + } + } + } + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ZipFileExtractToDirectory.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ZipFileExtractToDirectory.cs new file mode 100644 index 000000000..bec92355c --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ZipFileExtractToDirectory.cs @@ -0,0 +1,86 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using System; +using System.IO; +using System.IO.Compression; + +namespace Microsoft.DotNet.Build.Tasks +{ + public sealed class ZipFileExtractToDirectory : BuildTask + { + /// + /// The path to the archive to be extracted. + /// + [Required] + public string SourceArchive { get; set; } + + /// + /// The path of the directory to extract into. + /// + [Required] + public string DestinationDirectory { get; set; } + + /// + /// Indicates if the destination directory should be overwritten if it already exists. + /// + public bool OverwriteDestination { get; set; } + + /// + /// File entries to include in the extraction. Entries are relative + /// paths inside the archive. If null or empty, all files are extracted. + /// + public ITaskItem[] Include { get; set; } + + public override bool Execute() + { + try + { + if (Directory.Exists(DestinationDirectory)) + { + if (OverwriteDestination) + { + Log.LogMessage(MessageImportance.Low, $"'{DestinationDirectory}' already exists, trying to delete before unzipping..."); + Directory.Delete(DestinationDirectory, recursive: true); + } + else + { + Log.LogWarning($"'{DestinationDirectory}' already exists. Did you forget to set '{nameof(OverwriteDestination)}' to true?"); + } + } + + Log.LogMessage(MessageImportance.High, "Decompressing '{0}' into '{1}'...", SourceArchive, DestinationDirectory); + Directory.CreateDirectory(Path.GetDirectoryName(DestinationDirectory)); + + using (ZipArchive archive = ZipFile.OpenRead(SourceArchive)) + { + if (Include?.Length > 0) + { + foreach (ITaskItem entryItem in Include) + { + ZipArchiveEntry entry = archive.GetEntry(entryItem.ItemSpec); + string destinationPath = Path.Combine(DestinationDirectory, entryItem.ItemSpec); + + Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)); + entry.ExtractToFile(destinationPath, overwrite: false); + } + } + else + { + archive.ExtractToDirectory(DestinationDirectory); + } + } + } + catch (Exception e) + { + // We have 2 log calls because we want a nice error message but we also want to capture the callstack in the log. + Log.LogError("An exception has occurred while trying to decompress '{0}' into '{1}'.", SourceArchive, DestinationDirectory); + Log.LogErrorFromException(e, /*show stack=*/ true, /*show detail=*/ true, DestinationDirectory); + return false; + } + return true; + } + } +} \ No newline at end of file diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/_._ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/_._ new file mode 100644 index 000000000..e69de29bb diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/SourceBuild.MSBuildSdkResolver/SourceBuild.MSBuildSdkResolver.csproj b/src/SourceBuild/tarball/content/tools-local/tasks/SourceBuild.MSBuildSdkResolver/SourceBuild.MSBuildSdkResolver.csproj new file mode 100755 index 000000000..f6fa859e9 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/SourceBuild.MSBuildSdkResolver/SourceBuild.MSBuildSdkResolver.csproj @@ -0,0 +1,20 @@ + + + + netstandard2.0 + $(DotNetSdkResolversDir)$(MSBuildProjectName)\$(MSBuildProjectName).dll + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/SourceBuild.MSBuildSdkResolver/SourceBuiltSdkResolver.cs b/src/SourceBuild/tarball/content/tools-local/tasks/SourceBuild.MSBuildSdkResolver/SourceBuiltSdkResolver.cs new file mode 100644 index 000000000..250c620bf --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/SourceBuild.MSBuildSdkResolver/SourceBuiltSdkResolver.cs @@ -0,0 +1,200 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks +{ + /// + /// Extends the SDK to handle "SOURCE_BUILD_SDK_*" override environment variables. Each override + /// should provide a set of 3 environment variables: + /// + /// SOURCE_BUILD_SDK_ID_EXAMPLE=Your.Sdk.Example + /// ID of the SDK nuget package to override. + /// + /// SOURCE_BUILD_SDK_DIR_EXAMPLE=/git/repo/bin/extracted/Your.Sdk.Example/ + /// Directory where the sdk/Sdk.props and/or sdk/Sdk.targets files are located. This should be + /// the directory where the override SDK package is extracted. + /// + /// SOURCE_BUILD_SDK_VERSION_EXAMPLE=1.0.0-source-built + /// (Optional) Version of the SDK package to use. This is informational. + /// + public class SourceBuiltSdkResolver : SdkResolver + { + public override string Name => nameof(SourceBuiltSdkResolver); + + public override int Priority => 0; + + public override SdkResult Resolve( + SdkReference sdkReference, + SdkResolverContext resolverContext, + SdkResultFactory factory) + { + string sdkDescription = sdkReference.Name; + if (!string.IsNullOrEmpty(sdkReference.Version)) + { + sdkDescription += $" {sdkReference.Version}"; + } + if (!string.IsNullOrEmpty(sdkReference.MinimumVersion)) + { + sdkDescription += $" (>= {sdkReference.MinimumVersion})"; + } + + SourceBuiltSdkOverride[] overrides = Environment.GetEnvironmentVariables() + .Cast() + .Select(SourceBuiltSdkOverride.Create) + .Where(o => o != null) + .ToArray(); + + void LogMessage(string message) + { + resolverContext.Logger.LogMessage($"[{Name}] {message}", MessageImportance.High); + } + + if (overrides.Any()) + { + string separator = overrides.Length == 1 ? " " : Environment.NewLine; + + LogMessage( + $"Looking for SDK {sdkDescription}. Detected config(s) in env:{separator}" + + string.Join(Environment.NewLine, overrides.Select(o => o.ToString()))); + } + + SourceBuiltSdkOverride[] matches = overrides + .Where(o => sdkReference.Name.Equals(o?.Id, StringComparison.OrdinalIgnoreCase)) + .ToArray(); + + var unresolvableReasons = new List(); + + if (matches.Length != 1) + { + unresolvableReasons.Add( + $"{matches.Length} overrides found for '{sdkReference.Name}'"); + } + else + { + SourceBuiltSdkOverride match = matches[0]; + string[] matchProblems = match.GetValidationErrors().ToArray(); + + if (matchProblems.Any()) + { + unresolvableReasons.Add($"Found match '{match.Group}' with problems:"); + unresolvableReasons.AddRange(matchProblems); + } + else + { + LogMessage($"Overriding {sdkDescription} with '{match.Group}'"); + + return factory.IndicateSuccess(match.SdkDir, match.Version); + } + } + + return factory.IndicateFailure(unresolvableReasons.Select(r => $"[{Name}] {r}")); + } + + /// + /// Takes a directory path (not ending in a separator) and determines if it is the "sdk" + /// directory inside a SDK package with a case-insensitive comparison. + /// + private static bool IsSdkDirectory(string path) + { + return Path.GetFileName(path).Equals("sdk", StringComparison.OrdinalIgnoreCase); + } + + private class SourceBuiltSdkOverride + { + private const string EnvPrefix = "SOURCE_BUILT_SDK_"; + private const string EnvId = EnvPrefix + "ID_"; + private const string EnvVersion = EnvPrefix + "VERSION_"; + private const string EnvDir = EnvPrefix + "DIR_"; + + public static SourceBuiltSdkOverride Create(DictionaryEntry entry) + { + if (entry.Key is string key && key.StartsWith(EnvId)) + { + // E.g. "ARCADE" from "SOURCE_BUILD_SDK_ID_ARCADE=Microsoft.DotNet.Arcade.Sdk". + string group = key.Substring(EnvId.Length); + + if (string.IsNullOrEmpty(group)) + { + return null; + } + + string id = entry.Value as string; + string version = Environment.GetEnvironmentVariable(EnvVersion + group); + string dir = Environment.GetEnvironmentVariable(EnvDir + group); + + if (string.IsNullOrEmpty(version)) + { + version = "1.0.0-source-built"; + } + + string sdkDir = null; + if (!string.IsNullOrEmpty(dir)) + { + sdkDir = Directory.EnumerateDirectories(dir).FirstOrDefault(IsSdkDirectory); + } + + return new SourceBuiltSdkOverride + { + Group = group, + Id = id, + Version = version, + Dir = dir, + SdkDir = sdkDir, + }; + } + return null; + } + + /// + /// Name of the environment variable group, used to associate the multiple env vars. + /// + public string Group { get; set; } + + /// + /// ID of the SDK package. + /// + public string Id { get; set; } + + /// + /// Version of the source-built SDK package to use instead. + /// + public string Version { get; set; } + + /// + /// Directory where the source-built SDK files are found, in extracted form. + /// + public string Dir { get; set; } + + /// + /// Directory where the Sdk.props/Sdk.targets files are found, inside Dir. + /// + public string SdkDir { get; set; } + + public override string ToString() => $"'{Group}' for '{Id}/{Version}' at '{Dir}'"; + + public IEnumerable GetValidationErrors() + { + if (string.IsNullOrEmpty(Id)) + { + yield return $"'{EnvId}{Group}' not specified."; + } + if (string.IsNullOrEmpty(Dir)) + { + yield return $"'{EnvDir}{Group}' not specified."; + } + else if (string.IsNullOrEmpty(SdkDir)) + { + yield return $"Didn't find any 'sdk' directory in SDK package dir '{Dir}'"; + } + } + } + } +}