diff --git a/CODEOWNERS b/CODEOWNERS index 75e43ef23..ef04bdf5b 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -4,3 +4,4 @@ # Snaps /src/snaps/ @rbhanda +/src/sourceBuild/ @dotnet/source-build-internal \ No newline at end of file diff --git a/eng/Build.props b/eng/Build.props index 917aa8993..4df683bab 100644 --- a/eng/Build.props +++ b/eng/Build.props @@ -1,8 +1,12 @@ - + + + + + diff --git a/eng/SourceBuild.Version.Details.xml b/eng/SourceBuild.Version.Details.xml new file mode 100644 index 000000000..b11f1229a --- /dev/null +++ b/eng/SourceBuild.Version.Details.xml @@ -0,0 +1,28 @@ + + + + + https://github.com/mono/linker + f2588193553431636b9853b0f87209fa395a72c5 + + linker + + + + + https://github.com/dotnet/arcade + a3377cccde8639089f99107e2ba5df2c8cbe6394 + + + + https://github.com/dotnet/source-build-reference-packages + def2e2c6dc5064319250e2868a041a3dc07f9579 + + + + https://github.com/dotnet/sourcelink + 4b584dbc392bb1aad49c2eb1ab84d8b489b6dccc + + + + diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 36f034b6f..00947c0b3 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -13,25 +13,25 @@ https://github.com/dotnet/windowsdesktop 52ba9d8c25d343dd68558dd97f7438ec56c45d21 - + https://github.com/dotnet/runtime - 2a43c07bb82113a029090730d8ca001a68b22c05 + 7833828914a42f8c99dfa6f18ccd47f99dc2b56e - + https://github.com/dotnet/runtime - 2a43c07bb82113a029090730d8ca001a68b22c05 + 7833828914a42f8c99dfa6f18ccd47f99dc2b56e - + https://github.com/dotnet/runtime - 2a43c07bb82113a029090730d8ca001a68b22c05 + 7833828914a42f8c99dfa6f18ccd47f99dc2b56e - + https://github.com/dotnet/runtime - 2a43c07bb82113a029090730d8ca001a68b22c05 + 7833828914a42f8c99dfa6f18ccd47f99dc2b56e - + https://github.com/dotnet/runtime - 2a43c07bb82113a029090730d8ca001a68b22c05 + 7833828914a42f8c99dfa6f18ccd47f99dc2b56e @@ -39,34 +39,34 @@ https://github.com/dotnet/core-setup 7d57652f33493fa022125b7f63aad0d70c52d810 - + https://github.com/dotnet/runtime - 2a43c07bb82113a029090730d8ca001a68b22c05 + 7833828914a42f8c99dfa6f18ccd47f99dc2b56e - + https://github.com/dotnet/aspnetcore - 13943a5b2e4051b0f478f59c560f197582ebcf0b + 7b28c2b572cf32cf3c874d637ca8337a3b459830 - + https://github.com/dotnet/aspnetcore - 13943a5b2e4051b0f478f59c560f197582ebcf0b + 7b28c2b572cf32cf3c874d637ca8337a3b459830 - + https://github.com/dotnet/aspnetcore - 13943a5b2e4051b0f478f59c560f197582ebcf0b + 7b28c2b572cf32cf3c874d637ca8337a3b459830 - + https://github.com/dotnet/aspnetcore - 13943a5b2e4051b0f478f59c560f197582ebcf0b + 7b28c2b572cf32cf3c874d637ca8337a3b459830 - + https://github.com/dotnet/aspnetcore - 13943a5b2e4051b0f478f59c560f197582ebcf0b + 7b28c2b572cf32cf3c874d637ca8337a3b459830 - + https://github.com/dotnet/aspnetcore - 13943a5b2e4051b0f478f59c560f197582ebcf0b + 7b28c2b572cf32cf3c874d637ca8337a3b459830 https://github.com/dotnet/test-templates @@ -81,19 +81,19 @@ 6898c1c70c2d14e9725ddab6e1ebe8084c4d7e27 - + https://github.com/dotnet/templating - b0ec3feb4ec50f8b243f97fd60960ece98586f2c + 07188d40e98070273f7757531ee10fb916d2c20a - + https://github.com/dotnet/sdk - 101e405e97bf57462272552fad7ee3d7f140a576 + 3bafc012cfae73dbca1aa4ccdb9d7d7c55187b49 - + https://github.com/dotnet/sdk - 101e405e97bf57462272552fad7ee3d7f140a576 + 3bafc012cfae73dbca1aa4ccdb9d7d7c55187b49 @@ -104,13 +104,18 @@ https://github.com/dotnet/wpf fedb4cd155ba8de2e414b2a4ecdd4c71c5c95036 - + https://github.com/dotnet/fsharp - f8d54670c81ec6e29d312867207a38d0dc3aa192 + 334fb20360e5aac33c1ba7ab21693186fc2350a6 - + + https://github.com/dotnet/fsharp + 334fb20360e5aac33c1ba7ab21693186fc2350a6 + + + https://github.com/microsoft/vstest - ad32654bc155a059fda263f0ec99b445058d84ea + 45f100641b3b14310f67941ee499c5e8f34c8cae https://github.com/mono/linker @@ -118,14 +123,14 @@ linker - + https://github.com/dotnet/roslyn - 707ed8045df07669ce859ac4c748f48cf0a491ae + 9d5ad74040a0bee9c976579ee73b796dd53a4fcb - + https://github.com/dotnet/msbuild - 2c37803a9286d4fdc3d57813e1fdbfe2cfdbf668 + 52c41519f09a530db49dc62a955706f6d26f7e68 @@ -143,14 +148,14 @@ - + https://github.com/dotnet/arcade - 28a6403ee97077256fcdc60f599f0ad9e38e3cfa + a68ec1edf328e737b31a09cb49e1929c28e91d0c - + https://github.com/dotnet/arcade - 28a6403ee97077256fcdc60f599f0ad9e38e3cfa + a68ec1edf328e737b31a09cb49e1929c28e91d0c https://github.com/dotnet/source-build-reference-packages diff --git a/eng/Versions.props b/eng/Versions.props index c7922fac2..305e83ecb 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -20,7 +20,7 @@ - 6.0.0-beta.21321.2 + 6.0.0-beta.21324.3 @@ -32,7 +32,7 @@ - 6.0.100-preview.7.21322.1 + 6.0.100-preview.7.21324.2 @@ -45,35 +45,35 @@ - 6.0.0-preview.7.21322.5 - 6.0.0-preview.7.21322.5 - 6.0.0-preview.7.21322.5 - 6.0.0-preview.7.21322.5 - 6.0.0-preview.7.21322.5 - 6.0.0-preview.7.21322.5 + 6.0.0-preview.7.21323.6 + 6.0.0-preview.7.21323.6 + 6.0.0-preview.7.21323.6 + 6.0.0-preview.7.21323.6 + 6.0.0-preview.7.21323.6 + 6.0.0-preview.7.21323.6 0.2.0 - 6.0.100-preview.7.21322.14 - 6.0.100-preview.7.21322.14 + 6.0.100-preview.7.21324.11 + 6.0.100-preview.7.21324.11 $(MicrosoftNETSdkPackageVersion) $(MicrosoftNETSdkPackageVersion) $(MicrosoftNETSdkPackageVersion) - 6.0.0-preview.7.21321.15 + 6.0.0-preview.7.21324.1 - 6.0.0-preview.7.21321.15 - 6.0.0-preview.7.21321.15 - 6.0.0-preview.7.21321.15 - 6.0.0-preview.7.21321.15 - 6.0.0-preview.7.21321.15 + 6.0.0-preview.7.21324.1 + 6.0.0-preview.7.21324.1 + 6.0.0-preview.7.21324.1 + 6.0.0-preview.7.21324.1 + 6.0.0-preview.7.21324.1 2.1.0 @@ -149,7 +149,16 @@ 2.2.0-beta.19072.10 2.0.0 - 17.0.0-preview-20210518-01 + 17.0.0-preview-20210623-25 + + + + + 15.7.179 + 15.7.179 + 0.1.0-6.0.100-bootstrap.3 diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 5619c7aae..7942ffaf4 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -378,7 +378,16 @@ function InitializeVisualStudioMSBuild([bool]$install, [object]$vsRequirements = } $msbuildVersionDir = if ([int]$vsMajorVersion -lt 16) { "$vsMajorVersion.0" } else { "Current" } - return $global:_MSBuildExe = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin\msbuild.exe" + + $local:BinFolder = Join-Path $vsInstallDir "MSBuild\$msbuildVersionDir\Bin" + $local:Prefer64bit = if ($vsRequirements.Prefer64bit) { $vsRequirements.Prefer64bit } else { $false } + if ($local:Prefer64bit -and (Test-Path(Join-Path $local:BinFolder "amd64"))) { + $global:_MSBuildExe = Join-Path $local:BinFolder "amd64\msbuild.exe" + } else { + $global:_MSBuildExe = Join-Path $local:BinFolder "msbuild.exe" + } + + return $global:_MSBuildExe } function InitializeVisualStudioEnvironmentVariables([string] $vsInstallDir, [string] $vsMajorVersion) { diff --git a/global.json b/global.json index 4914d416a..5998afb3f 100644 --- a/global.json +++ b/global.json @@ -11,7 +11,7 @@ "cmake": "3.16.4" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21321.2", + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21324.3", "Microsoft.DotNet.CMake.Sdk": "6.0.0-beta.21253.2" } } diff --git a/src/SourceBuild/Arcade/README.md b/src/SourceBuild/Arcade/README.md new file mode 100644 index 000000000..914a87b3f --- /dev/null +++ b/src/SourceBuild/Arcade/README.md @@ -0,0 +1 @@ +The source and targets in the `src/SourceBuild/Arcade` directory are intended to move into the Arcade repo at some point. They are added here for ease of development while developing the tarball generation process. See https://github.com/dotnet/source-build/issues/2295 \ No newline at end of file diff --git a/src/SourceBuild/Arcade/src/SourceBuild.Tasks.csproj b/src/SourceBuild/Arcade/src/SourceBuild.Tasks.csproj new file mode 100644 index 000000000..735289e70 --- /dev/null +++ b/src/SourceBuild/Arcade/src/SourceBuild.Tasks.csproj @@ -0,0 +1,17 @@ + + + net5.0 + false + false + disable + + + + + + + + + + + diff --git a/src/SourceBuild/Arcade/src/Tarball_ReadSourceBuildIntermediateNupkgDependencies.cs b/src/SourceBuild/Arcade/src/Tarball_ReadSourceBuildIntermediateNupkgDependencies.cs new file mode 100644 index 000000000..fbd497b72 --- /dev/null +++ b/src/SourceBuild/Arcade/src/Tarball_ReadSourceBuildIntermediateNupkgDependencies.cs @@ -0,0 +1,135 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System.Collections.Generic; +using System.Linq; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks +{ + /// + /// Reads entries in a Version.Details.xml file to find intermediate nupkg dependencies. For + /// each dependency with a "SourceBuild" element, adds an item to the "Dependencies" output. + /// + public class Tarball_ReadSourceBuildIntermediateNupkgDependencies : Task + { + [Required] + public string VersionDetailsXmlFile { get; set; } + + [Required] + public string SourceBuildIntermediateNupkgPrefix { get; set; } + + /// + /// The intermediate nupkg RID to use if any RID-specific intermediate nupkgs are required. + /// If this parameter isn't specified, RID-specific intermediate nupkgs can't be used and + /// this task fails. + /// + public string SourceBuildIntermediateNupkgRid { get; set; } + + /// + /// %(Identity): NuGet package ID. + /// %(Name): The Name of the dependency from Version.Details.xml. + /// %(ExactVersion): NuGet package version. This can be used to look up the restored package + /// contents in a package cache. + /// %(Version): NuGet package version, wrapped in "[version]" syntax for exact match. + /// %(Uri): The URI for the repo. + /// %(Sha): The commit Sha for the dependency. + /// %(SourceBuildRepoName): The repo name to use in source-build. + /// + [Output] + public ITaskItem[] Dependencies { get; set; } + + public override bool Execute() + { + XElement root = XElement.Load(VersionDetailsXmlFile, LoadOptions.PreserveWhitespace); + + XName CreateQualifiedName(string plainName) + { + return root.GetDefaultNamespace().GetName(plainName); + } + + Dependencies = root + .Elements() + .Elements(CreateQualifiedName("Dependency")) + .Select(d => + { + XElement sourceBuildElement = d.Element(CreateQualifiedName("SourceBuild")); + + if (sourceBuildElement == null) + { + // Ignore element: doesn't represent a source-build dependency. + return null; + } + + string repoName = sourceBuildElement.Attribute("RepoName")?.Value; + + if (string.IsNullOrEmpty(repoName)) + { + Log.LogError($"Dependency SourceBuild RepoName null or empty in '{VersionDetailsXmlFile}' element {d}"); + return null; + } + + string dependencyName = d.Attribute("Name")?.Value ?? string.Empty; + + if (string.IsNullOrEmpty(dependencyName)) + { + // Log name missing as FYI, but this is not an error case for source-build. + Log.LogMessage($"Dependency Name null or empty in '{VersionDetailsXmlFile}' element {d}"); + } + + string dependencyVersion = d.Attribute("Version")?.Value; + + string uri = d.Element(CreateQualifiedName("Uri"))?.Value; + string sha = d.Element(CreateQualifiedName("Sha"))?.Value; + string sourceBuildRepoName = sourceBuildElement.Attribute("RepoName")?.Value; + + if (string.IsNullOrEmpty(dependencyVersion)) + { + // We need a version to bring down an intermediate nupkg. Fail. + Log.LogError($"Dependency Version null or empty in '{VersionDetailsXmlFile}' element {d}"); + return null; + } + + string identity = SourceBuildIntermediateNupkgPrefix + repoName; + + bool.TryParse( + sourceBuildElement.Attribute("ManagedOnly")?.Value, + out bool managedOnly); + + // If RID-specific, add the RID to the end of the identity. + if (!managedOnly) + { + if (string.IsNullOrEmpty(SourceBuildIntermediateNupkgRid)) + { + Log.LogError( + $"Parameter {nameof(SourceBuildIntermediateNupkgRid)} was " + + "not specified, indicating this project depends only on managed " + + "inputs. However, source-build element is not ManagedOnly: " + + sourceBuildElement); + return null; + } + + identity += "." + SourceBuildIntermediateNupkgRid; + } + + return new TaskItem( + identity, + new Dictionary + { + ["Name"] = dependencyName, + ["Version"] = $"[{dependencyVersion}]", + ["ExactVersion"] = dependencyVersion, + ["Uri"] = uri, + ["Sha"] = sha, + ["SourceBuildRepoName"] = sourceBuildRepoName + }); + }) + .Where(d => d != null) + .ToArray(); + + return !Log.HasLoggedErrors; + } + } +} diff --git a/src/SourceBuild/Arcade/src/Tarball_WriteSourceRepoProperties.cs b/src/SourceBuild/Arcade/src/Tarball_WriteSourceRepoProperties.cs new file mode 100644 index 000000000..58dbab586 --- /dev/null +++ b/src/SourceBuild/Arcade/src/Tarball_WriteSourceRepoProperties.cs @@ -0,0 +1,204 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Tasks; +using Microsoft.Build.Utilities; +using NuGet.Versioning; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml.Serialization; + +namespace Microsoft.DotNet.SourceBuild.Tasks +{ + /// + /// Writes a props file to the given directory for each dependency specified + /// plus adds or updates an existing props file with all dependencies. The + /// intention is for the props file to be included by a source-build build + /// to get metadata about each dependent repo. + /// + public class Tarball_WriteSourceRepoProperties : Task + { + /// + /// The directory to write the props files to. + /// + [Required] + public string SourceBuildMetadataDir { get; set; } + + /// + /// Dependencies to include in the props files. + /// + /// %(Identity): NuGet package ID. + /// %(Name): The Name of the dependency from Version.Details.xml. + /// %(ExactVersion): NuGet package version. This can be used to look up the restored package + /// contents in a package cache. + /// %(Version): NuGet package version, wrapped in "[version]" syntax for exact match. + /// %(Uri): The URI for the repo. + /// %(Sha): The commit Sha for the dependency. + /// %(SourceBuildRepoName): The repo name to use in source-build. + /// + /// + [Required] + public ITaskItem[] Dependencies { get; set; } + + public override bool Execute() + { + var allRepoProps = new Dictionary(); + + foreach (var dependency in Dependencies.Select(dep => + new { + Name = dep.GetMetadata("Name"), + Version = dep.GetMetadata("ExactVersion"), + Sha = dep.GetMetadata("Sha"), + Uri = dep.GetMetadata("Uri") + })) + { + string repoName = GetDefaultRepoNameFromUrl(dependency.Uri); + string safeRepoName = repoName.Replace("-", ""); + string propsPath = Path.Combine(SourceBuildMetadataDir, $"{repoName}.props"); + DerivedVersion derivedVersion = GetVersionInfo(dependency.Version, "0"); + var repoProps = new Dictionary + { + ["GitCommitHash"] = dependency.Sha, + ["OfficialBuildId"] = derivedVersion.OfficialBuildId, + ["OutputPackageVersion"] = dependency.Version, + ["PreReleaseVersionLabel"] = derivedVersion.PreReleaseVersionLabel, + ["IsStable"] = string.IsNullOrWhiteSpace(derivedVersion.PreReleaseVersionLabel) ? "true" : "false", + }; + WritePropsFile(propsPath, repoProps); + allRepoProps[$"{safeRepoName}GitCommitHash"] = dependency.Sha; + allRepoProps[$"{safeRepoName}OutputPackageVersion"] = dependency.Version; + } + string allRepoPropsPath = Path.Combine(SourceBuildMetadataDir, "AllRepoVersions.props"); + Log.LogMessage(MessageImportance.Normal, $"[{DateTimeOffset.Now}] Writing all repo versions to {allRepoPropsPath}"); + UpdatePropsFile(allRepoPropsPath, allRepoProps); + + return !Log.HasLoggedErrors; + } + + /// + /// Reverse a version in the Arcade style (https://github.com/dotnet/arcade/blob/fb92b14d8cd07cf44f8f7eefa8ac58d7ffd05f3f/src/Microsoft.DotNet.Arcade.Sdk/tools/Version.BeforeCommonTargets.targets#L18) + /// back to an OfficialBuildId + ReleaseLabel which we can then supply to get the same resulting version number. + /// + /// The complete version, e.g. 1.0.0-beta1-19720.5 + /// The current commit count of the repo. This is used for some repos that do not use the standard versioning scheme. + /// + private static DerivedVersion GetVersionInfo(string version, string commitCount) + { + var nugetVersion = new NuGetVersion(version); + + if (!string.IsNullOrWhiteSpace(nugetVersion.Release)) + { + var releaseParts = nugetVersion.Release.Split('-', '.'); + if (releaseParts.Length == 2) + { + if (releaseParts[1].TrimStart('0') == commitCount) + { + // core-sdk does this - OfficialBuildId is only used for their fake package and not in anything shipped + return new DerivedVersion { OfficialBuildId = DateTime.Now.ToString("yyyyMMdd.1"), PreReleaseVersionLabel = releaseParts[0] }; + } + else + { + // NuGet does this - arbitrary build IDs + return new DerivedVersion { OfficialBuildId = releaseParts[1], PreReleaseVersionLabel = releaseParts[0] }; + } + } + else if (releaseParts.Length == 3) + { + // VSTest uses full dates for the first part of their preview build numbers + if (DateTime.TryParseExact(releaseParts[1], "yyyyMMdd", new CultureInfo("en-US"), DateTimeStyles.AssumeLocal, out DateTime fullDate)) + { + return new DerivedVersion { OfficialBuildId = $"{releaseParts[1]}.{releaseParts[2]}", PreReleaseVersionLabel = releaseParts[0] }; + } + else if (int.TryParse(releaseParts[1], out int datePart) && int.TryParse(releaseParts[2], out int buildPart)) + { + if (datePart > 1 && datePart < 8 && buildPart > 1000 && buildPart < 10000) + { + return new DerivedVersion { OfficialBuildId = releaseParts[2], PreReleaseVersionLabel = $"{releaseParts[0]}.{releaseParts[1]}" }; + } + else + { + return new DerivedVersion { OfficialBuildId = $"20{((datePart / 1000))}{((datePart % 1000) / 50):D2}{(datePart % 50):D2}.{buildPart}", PreReleaseVersionLabel = releaseParts[0] }; + } + } + } + else if (releaseParts.Length == 4) + { + // new preview version style, e.g. 5.0.0-preview.7.20365.12 + if (int.TryParse(releaseParts[2], out int datePart) && int.TryParse(releaseParts[3], out int buildPart)) + { + return new DerivedVersion { OfficialBuildId = $"20{((datePart / 1000))}{((datePart % 1000) / 50):D2}{(datePart % 50):D2}.{buildPart}", PreReleaseVersionLabel = $"{releaseParts[0]}.{releaseParts[1]}" }; + } + } + } + else + { + // finalized version number (x.y.z) - probably not our code + // VSTest, Application Insights, Newtonsoft.Json do this + return new DerivedVersion { OfficialBuildId = DateTime.Now.ToString("yyyyMMdd.1"), PreReleaseVersionLabel = string.Empty }; + } + + throw new FormatException($"Can't derive a build ID from version {version} (commit count {commitCount}, release {string.Join(";", nugetVersion.Release.Split('-', '.'))})"); + } + + private static string GetDefaultRepoNameFromUrl(string repoUrl) + { + if (repoUrl.EndsWith(".git")) + { + repoUrl = repoUrl.Substring(0, repoUrl.Length - ".git".Length); + } + return repoUrl.Substring(repoUrl.LastIndexOf("/") + 1); + } + + private static void UpdatePropsFile(string filePath, Dictionary properties) + { + if (!File.Exists(filePath)) + { + WritePropsFile(filePath, properties); + } + else + { + var content = new StringBuilder(); + foreach (var line in File.ReadAllLines(filePath)) + { + content.AppendLine(line); + if (line.Contains("")) + { + foreach (var propName in properties.Keys.OrderBy(k => k)) + { + content.AppendLine($" <{propName}>{properties[propName]}"); + } + } + } + File.WriteAllText(filePath, content.ToString()); + } + } + + private static void WritePropsFile(string filePath, Dictionary properties) + { + var content = new StringBuilder(); + content.AppendLine(""); + content.AppendLine(""); + content.AppendLine(" "); + foreach (var propName in properties.Keys.OrderBy(k => k)) + { + content.AppendLine($" <{propName}>{properties[propName]}"); + } + content.AppendLine(" "); + content.AppendLine(""); + File.WriteAllText(filePath, content.ToString()); + } + + private class DerivedVersion + { + internal string OfficialBuildId { get; set; } + internal string PreReleaseVersionLabel { get; set; } + } + } +} diff --git a/src/SourceBuild/Arcade/tools/BuildTasks.props b/src/SourceBuild/Arcade/tools/BuildTasks.props new file mode 100644 index 000000000..4014b87ad --- /dev/null +++ b/src/SourceBuild/Arcade/tools/BuildTasks.props @@ -0,0 +1,7 @@ + + + + + $(RepoRoot)\artifacts\bin\SourceBuild.Tasks\$(Configuration)\net5.0\SourceBuild.Tasks.dll + + \ No newline at end of file diff --git a/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets b/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets new file mode 100644 index 000000000..914cf1ad3 --- /dev/null +++ b/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets @@ -0,0 +1,179 @@ + + + + + + + + + + + + + + $(RepoRoot)artifacts/tarball/ + $([MSBuild]::EnsureTrailingSlash('$(TarballDir)')) + $(TarballRootDir)src/ + $(TarballRootDir)git-info/ + quiet + + + + + + + + + + + + + + + + + + + + + + + + + + $(GitHubRepositoryName) + 1.0.0 + 1.0.0 + @(RootRepoCommitSha) + @(RootRepoUri) + $(GitHubRepositoryName) + + + + + + + %(SourceBuildRepos.SourceBuildRepoName) + %(SourceBuildRepos.Uri) + %(SourceBuildRepos.Sha) + + + + + + + + + + + + + $(SourceBuildRepoName).$(RepoSha)/ + $(TarballSourceDir)$(SourceDir) + $(TarballSourceDir)$(SourceDir)eng/ + $(TarballRepoSourceEngDir)Version.Details.xml + -q + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(TarballRootDir)packages/archive/ + Private.SourceBuilt.Artifacts + https://dotnetcli.azureedge.net/source-built-artifacts/assets/ + archiveArtifacts.txt + $(ExternalTarballsDir)$(ArchiveArtifactsTextFileName) + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/BuildSourceBuildTarball.proj b/src/SourceBuild/tarball/BuildSourceBuildTarball.proj new file mode 100644 index 000000000..147239862 --- /dev/null +++ b/src/SourceBuild/tarball/BuildSourceBuildTarball.proj @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/src/SourceBuild/tarball/content/Directory.Build.props b/src/SourceBuild/tarball/content/Directory.Build.props new file mode 100644 index 000000000..9c556df3f --- /dev/null +++ b/src/SourceBuild/tarball/content/Directory.Build.props @@ -0,0 +1,268 @@ + + + + false + <_SuppressSdkImports>true + Release + + + + + + netstandard2.0 + + false + + + + + $([System.Runtime.InteropServices.RuntimeInformation]::ProcessArchitecture.ToString().ToLowerInvariant()) + $(BuildArchitecture) + x64 + + true + + + false + false + true + false + + + + + false + + + + $(MSBuildThisFileDirectory) + $(ProjectDir)targets/ + $(ProjectDir)keys/ + $([MSBuild]::EnsureTrailingSlash('$(CustomDotNetSdkDir)')) + $([MSBuild]::NormalizeDirectory('$(DOTNET_INSTALL_DIR)')) + $(ProjectDir).dotnet/ + $(DotNetCliToolDir)dotnet + $(ProjectDir)patches/ + $(NuGetPackageRoot) + $(ProjectDir)packages/restored/ + $(PackagesDir)ArcadeBootstrapPackage/ + + $(NETCoreSdkVersion) + $(DotNetCliToolDir)sdk/$(SDK_VERSION)/ + $(DotNetSdkDir)SdkResolvers/ + $(DotNetCliToolDir)sdk/$(SDK_VERSION)/ + + + + + false + + + + minimal + + + + false + + + + .cmd + .sh + .zip + .tar.gz + + + + $(ProjectDir)src/ + $(ProjectDir).gitmodules + $(ProjectDir)ProdConFeed.txt + + + + + $(ProjectDir)artifacts/ + $(ProjectDir)eng/ + + + + $(ArtifactsDir) + $(BaseOutputPath)src/ + $(ProjectDir).git/modules/src/ + $(ProjectDir)Tools/ + $(ToolsDir)source-built/ + $(ProjectDir)tools-local/ + $(ToolsLocalDir)tasks/ + + $(TaskDirectory)Microsoft.DotNet.SourceBuild.Tasks.XPlat/bin/$(Configuration)/ + $(XPlatTasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.XPlat.dll + + $(TaskDirectory)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/ + $(LeakDetectionTasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.dll + + $(BaseOutputPath)obj/ + $(BaseOutputPath)$(Platform)/$(Configuration)/ + $(BaseIntermediatePath)$(Platform)/$(Configuration)/ + $(IntermediatePath)blobs/ + $(IntermediatePath)nuget-packages/ + $(IntermediatePath)blob-feed/ + $(SourceBuiltBlobFeedDir)packages/ + $(SourceBuiltBlobFeedDir)assets/ + $(ProjectDir)packages/prebuilt/ + $(ProjectDir)packages/previouslyRestored/ + $(ProjectDir)packages/source-built/ + $(CustomPrebuiltSourceBuiltPackagesPath)/ + $(OutputPath) + $(LocalBlobStorageRoot)Sdk/ + $(LocalBlobStorageRoot)Runtime/ + $(LocalBlobStorageRoot)aspnetcore/Runtime/ + $(IntermediatePath)RestoreSources.props + $(IntermediatePath)PackageVersions.props + $(IntermediatePath)GennedPackageVersions.props + $(IntermediatePath)PackageVersions.props + $(BaseOutputPath)logs/ + $(BaseOutputPath)msbuild-debug/ + $(BaseOutputPath)roslyn-debug/ + $(BaseOutputPath)aspnet-debug + $(AspNetRazorBuildServerLogDir)razor-build-server.log + + $(BaseOutputPath)git-info/ + + $(ProjectDir)git-info/ + $(GitInfoOutputDir)$(RepositoryName).props + $(GitInfoOutputDir)AllRepoVersions.props + $(GitInfoOfflineDir)$(RepositoryName).props + $(GitInfoOfflineDir)AllRepoVersions.props + $(BaseOutputPath)prebuilt-report/ + $(PackageReportDir)prebuilt-usage.xml + $(PackageReportDir)poison-usage.xml + $(PackageReportDir)poison-catalog.xml + .prebuilt.xml + $(PackageReportDir)poison-source-built-catalog.xml + .source-built.xml + $(PackageReportDir)all-project-assets-json-files.zip + $(PackageReportDir)prodcon-build.xml + $(PackageReportDir)poisoned.txt + $(BaseOutputPath)conflict-report/ + $(PackageReportDir)PrebuiltBurndownData-offline.csv + $(PackageReportDir)PrebuiltBurndownData-online.csv + $(IntermediatePath)reference-packages/ + $(IntermediatePath)text-only-packages/ + $(IntermediatePath)external-tarballs/ + + $(ProjectDir)packages/archive/ + $(ProjectDir)packages/reference/ + $(ProjectDir)packages/text-only/ + $(ReferencePackagesBaseDir)packages/ + Private.SourceBuilt.Artifacts + https://dotnetcli.azureedge.net/source-built-artifacts/assets/ + archiveArtifacts.txt + $(ExternalTarballsDir)$(ArchiveArtifactsTextFileName) + $(ToolsLocalDir)prebuilt-baseline- + $(BaselineDataFile)offline.xml + $(BaselineDataFile)online.xml + $(OfflineBaselineDataFile) + $(OnlineBaselineDataFile) + + $(ProjectDir)test/exclusions/ + + + + + + + ROOTFS_DIR=$(BaseIntermediatePath)crossrootfs/arm + ROOTFS_DIR=$(BaseIntermediatePath)crossrootfs/armel + + + + $([System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier) + + Windows_NT + OSX + Linux + FreeBSD + + + + freebsd-$(Platform) + osx-$(Platform) + linux-$(Platform) + win-$(Platform) + + + + + known-good + known-good-tests + + + + + $(BaseIntermediatePath)semaphores/ + + + + + source + 30000001-1 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/Directory.Build.targets b/src/SourceBuild/tarball/content/Directory.Build.targets new file mode 100644 index 000000000..c9cd51f95 --- /dev/null +++ b/src/SourceBuild/tarball/content/Directory.Build.targets @@ -0,0 +1,63 @@ + + + + + + + + + + + + + + + + + + + + $(IgnoredRepos);https://dev.azure.com/dnceng/internal/_git/dotnet-optimization + $(IgnoredRepos);https://dev.azure.com/devdiv/DevDiv/_git/DotNet-Trusted + $(IgnoredRepos);https://devdiv.visualstudio.com/DevDiv/_git/DotNet-Trusted + $(IgnoredRepos);https://dnceng@dev.azure.com/dnceng/internal/_git/dotnet-optimization + $(IgnoredRepos);https://dev.azure.com/dnceng/internal/_git/dotnet-core-setup + $(IgnoredRepos);https://github.com/dotnet/source-build-reference-packages + + $(ClonedSubmoduleDirectory) + + $(DarcCloneArguments) --git-dir-folder $(ClonedSubmoduleGitRootDirectory) + $(DarcCloneArguments) --include-toolset + $(DarcCloneArguments) --ignore-repos "$(IgnoredRepos)" + $(DarcCloneArguments) --debug + + + bogus + $(DarcCloneArguments) --azdev-pat $(AzDoPat) + $(DarcCloneArguments) --github-pat bogus + + + $(DarcCloneArguments) --depth 0 + + $(DotNetCliToolDir)dotnet $(DarcDll) clone $(DarcCloneArguments) + + + + + + + + $([System.IO.File]::ReadAllText('$(ProdConFeedPath)').Trim()) + $(ProdConBlobFeedUrl.Replace('https://dotnetfeed.blob.core.windows.net/', '$(ProdConBlobFeedUrlPrefix)')) + + + + diff --git a/src/SourceBuild/tarball/content/build.proj b/src/SourceBuild/tarball/content/build.proj new file mode 100644 index 000000000..c8f1995ce --- /dev/null +++ b/src/SourceBuild/tarball/content/build.proj @@ -0,0 +1,197 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + N/A + + + + + + + + + + N/A + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ./smoke-test.sh + $(SmokeTestCommand) --minimal + $(SmokeTestCommand) --projectOutput + $(SmokeTestCommand) --configuration $(Configuration) + $(SmokeTestCommand) --archiveRestoredPackages + + + $(SmokeTestCommand) --excludeWebHttpsTests + + + + + + + + + + + + + + + + + + + $(RelativeBlobPath)/%(Filename)%(Extension) + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/build.sh b/src/SourceBuild/tarball/content/build.sh new file mode 100755 index 000000000..13f8ccc7b --- /dev/null +++ b/src/SourceBuild/tarball/content/build.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +usage() { + echo "usage: $0 [options]" + echo "options:" + echo " --with-ref-packages use the specified directory of reference packages" + echo " --with-packages use the specified directory of previously-built packages" + echo " --with-sdk use the SDK in the specified directory for bootstrapping" + echo "use -- to send the remaining arguments to MSBuild" + echo "" +} + +SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)" + +MSBUILD_ARGUMENTS=("/p:OfflineBuild=true" "/flp:v=detailed") +CUSTOM_REF_PACKAGES_DIR='' +CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR='' +alternateTarget=false +CUSTOM_SDK_DIR='' + +while :; do + if [ $# -le 0 ]; then + break + fi + + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + --run-smoke-test) + alternateTarget=true + MSBUILD_ARGUMENTS+=( "/t:RunSmokeTest" ) + ;; + --with-packages) + CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR="$(cd -P "$2" && pwd)" + if [ ! -d "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" ]; then + echo "Custom prviously built packages directory '$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR' does not exist" + exit 1 + fi + MSBUILD_ARGUMENTS+=( "/p:CustomPrebuiltSourceBuiltPackagesPath=$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" ) + shift + ;; + --with-sdk) + CUSTOM_SDK_DIR="$(cd -P "$2" && pwd)" + if [ ! -d "$CUSTOM_SDK_DIR" ]; then + echo "Custom SDK directory '$CUSTOM_SDK_DIR' does not exist" + exit 1 + fi + if [ ! -x "$CUSTOM_SDK_DIR/dotnet" ]; then + echo "Custom SDK '$CUSTOM_SDK_DIR/dotnet' does not exist or is not executable" + exit 1 + fi + shift + ;; + --) + shift + echo "Detected '--': passing remaining parameters '$@' as build.sh arguments." + break + ;; + -?|-h|--help) + usage + exit 0 + ;; + *) + echo "Unrecognized argument '$1'" + usage + exit 1 + ;; + esac + shift +done + +if [ -f "$SCRIPT_ROOT/packages/archive/archiveArtifacts.txt" ]; then + ARCHIVE_ERROR=0 + if [ ! -d "$SCRIPT_ROOT/.dotnet" ] && [ "$CUSTOM_SDK_DIR" == "" ]; then + echo "ERROR: SDK not found at $SCRIPT_ROOT/.dotnet" + ARCHIVE_ERROR=1 + fi + if [ ! -f $SCRIPT_ROOT/packages/archive/Private.SourceBuilt.Artifacts*.tar.gz ] && [ "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR" == "" ]; then + echo "ERROR: Private.SourceBuilt.Artifacts artifact not found at $SCRIPT_ROOT/packages/archive/ - Either run prep.sh or pass --with-packages parameter" + ARCHIVE_ERROR=1 + fi + if [ $ARCHIVE_ERROR == 1 ]; then + echo "" + echo " Errors detected in tarball. To prep the tarball, run prep.sh while online to install an SDK" + echo " and Private.SourceBuilt.Artifacts tarball. After prepping the tarball, the tarball can be" + echo " built offline. As an alternative to prepping the tarball, these assets can be provided using" + echo " the --with-sdk and --with-packages parameters" + exit 1 + fi +fi + +if [ -d "$CUSTOM_SDK_DIR" ]; then + export SDK_VERSION=`"$CUSTOM_SDK_DIR/dotnet" --version` + export CLI_ROOT="$CUSTOM_SDK_DIR" + export _InitializeDotNetCli="$CLI_ROOT/dotnet" + export CustomDotNetSdkDir="$CLI_ROOT" + echo "Using custom bootstrap SDK from '$CLI_ROOT', version '$SDK_VERSION'" +else + sdkLine=`grep -m 1 'dotnet' "$SCRIPT_ROOT/global.json"` + sdkPattern="\"dotnet\" *: *\"(.*)\"" + if [[ $sdkLine =~ $sdkPattern ]]; then + export SDK_VERSION=${BASH_REMATCH[1]} + export CLI_ROOT="$SCRIPT_ROOT/.dotnet" + fi +fi + +packageVersionsPath='' +restoredPackagesDir="$SCRIPT_ROOT/packages/restored" + +if [ -d "$SCRIPT_ROOT/packages/archive" ]; then + sourceBuiltArchive=`find $SCRIPT_ROOT/packages/archive -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'` + if [ -f "$sourceBuiltArchive" ]; then + tar -xzf "$sourceBuiltArchive" -C /tmp PackageVersions.props + packageVersionsPath=/tmp/PackageVersions.props + fi +else + if [ -f "$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR/PackageVersions.props" ]; then + packageVersionsPath="$CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR/PackageVersions.props" + fi +fi + +if [ ! -f "$packageVersionsPath" ]; then + echo "Cannot find PackagesVersions.props. Debugging info:" + echo " Attempted archive path: $SCRIPT_ROOT/packages/archive" + echo " Attempted custom PVP path: $CUSTOM_PREVIOUSLY_BUILT_PACKAGES_DIR/PackageVersions.props" + exit 1 +fi + +arcadeSdkLine=`grep -m 1 'MicrosoftDotNetArcadeSdkVersion' "$packageVersionsPath"` +versionPattern="(.*)" +if [[ $arcadeSdkLine =~ $versionPattern ]]; then + export ARCADE_BOOTSTRAP_VERSION=${BASH_REMATCH[1]} + + # Ensure that by default, the bootstrap version of the Arcade SDK is used. Source-build infra + # projects use bootstrap Arcade SDK, and would fail to find it in the tarball build. The repo + # projects overwrite this so that they use the source-built Arcade SDK instad. + export SOURCE_BUILT_SDK_ID_ARCADE=Microsoft.DotNet.Arcade.Sdk + export SOURCE_BUILT_SDK_VERSION_ARCADE=$ARCADE_BOOTSTRAP_VERSION + export SOURCE_BUILT_SDK_DIR_ARCADE=$restoredPackagesDir/ArcadeBootstrapPackage/microsoft.dotnet.arcade.sdk/$ARCADE_BOOTSTRAP_VERSION +fi + +sourceLinkLine=`grep -m 1 'MicrosoftSourceLinkCommonVersion' "$packageVersionsPath"` +versionPattern="(.*)" +if [[ $sourceLinkLine =~ $versionPattern ]]; then + export SOURCE_LINK_BOOTSTRAP_VERSION=${BASH_REMATCH[1]} +fi + +echo "Found bootstrap SDK $SDK_VERSION, bootstrap Arcade $ARCADE_BOOTSTRAP_VERSION, bootstrap SourceLink $SOURCE_LINK_BOOTSTRAP_VERSION" + +export DOTNET_CLI_TELEMETRY_OPTOUT=1 +export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 +export NUGET_PACKAGES=$restoredPackagesDir/ + +if [ "$alternateTarget" == "true" ]; then + "$CLI_ROOT/dotnet" $CLI_ROOT/sdk/$SDK_VERSION/MSBuild.dll "$SCRIPT_ROOT/build.proj" /bl:source-build-test.binlog /clp:v=m ${MSBUILD_ARGUMENTS[@]} "$@" +else + LogDateStamp=$(date +"%m%d%H%M%S") + $CLI_ROOT/dotnet $CLI_ROOT/sdk/$SDK_VERSION/MSBuild.dll /bl:$SCRIPT_ROOT/artifacts/log/Debug/BuildXPlatTasks_$LogDateStamp.binlog $SCRIPT_ROOT/tools-local/init-build.proj /t:PrepareOfflineLocalTools ${MSBUILD_ARGUMENTS[@]} "$@" + + $CLI_ROOT/dotnet $CLI_ROOT/sdk/$SDK_VERSION/MSBuild.dll /bl:$SCRIPT_ROOT/artifacts/log/Debug/Build_$LogDateStamp.binlog $SCRIPT_ROOT/build.proj ${MSBUILD_ARGUMENTS[@]} "$@" +fi diff --git a/src/SourceBuild/tarball/content/eng/Build.props b/src/SourceBuild/tarball/content/eng/Build.props new file mode 100644 index 000000000..d8414bdbf --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/Build.props @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/eng/No.proj b/src/SourceBuild/tarball/content/eng/No.proj new file mode 100644 index 000000000..9ba857dd2 --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/No.proj @@ -0,0 +1,9 @@ + + + + + + diff --git a/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.AllowedPrebuilts.targets b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.AllowedPrebuilts.targets new file mode 100644 index 000000000..f4191b490 --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.AllowedPrebuilts.targets @@ -0,0 +1,11 @@ + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.DestructiveCleanup.targets b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.DestructiveCleanup.targets new file mode 100644 index 000000000..18ef94a13 --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.DestructiveCleanup.targets @@ -0,0 +1,63 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.KnownExtraPrebuilts.targets b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.KnownExtraPrebuilts.targets new file mode 100644 index 000000000..fc10206fc --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.KnownExtraPrebuilts.targets @@ -0,0 +1,136 @@ + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.TextOnlyPrebuilts.targets b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.TextOnlyPrebuilts.targets new file mode 100644 index 000000000..efc0276b4 --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.TextOnlyPrebuilts.targets @@ -0,0 +1,99 @@ + + + + + + + + $([MSBuild]::NormalizeDirectory('$(TarballRootDir)', 'packages', 'text-only')) + + + + + + + + + + + + + + + + + + + + $([System.IO.Path]::GetDirectoryName('$(TextOnlyPackageRootDir)')) + $([System.IO.Path]::GetDirectoryName('$(PackageVersionDirFile)')) + $([System.IO.Path]::GetFileName('$(PackageVersionDirFile)')) + $([System.IO.Path]::GetFileName('$(PackageIdDirFile)')) + + $([MSBuild]::NormalizeDirectory('$(TextOnlyPackageBaseDir)', '$(PackageId)', '$(PackageVersion)')) + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.targets b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.targets new file mode 100644 index 000000000..3948a9cc6 --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/SourceBuild.Tarball.targets @@ -0,0 +1,386 @@ + + + + + + + + + + + + + $([MSBuild]::NormalizeDirectory('$(TarballRoot)')) + $([MSBuild]::NormalizeDirectory('$(TarballRootDir)', 'src')) + $([MSBuild]::NormalizeDirectory('$(TarballRootDir)', 'packages', 'prebuilt')) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(TarballRootDir)source-build-info.txt + + + + + + + + $(ProjectDir)tools-local\tasks\Microsoft.DotNet.SourceBuild.Tasks.LeakDetection\Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj + + + + + + + + + + + + + + + + + + + + + + + + Detected forbidden prebuilts. They must be removed, or explicitly allowed (see target for details): + $(PrebuiltErrorText)%0A@(ForbiddenPrebuiltPackageFile -> '%(Identity)', '%0A') + + + + + + diff --git a/src/SourceBuild/tarball/content/eng/Versions.props b/src/SourceBuild/tarball/content/eng/Versions.props new file mode 100644 index 000000000..d8bab4660 --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/Versions.props @@ -0,0 +1,26 @@ + + + + + 6 + 0 + 0-preview.6 + 100-preview.6 + $(MajorVersion).$(MinorVersion).$(RuntimePatchVersion) + $(MajorVersion).$(MinorVersion).$(RuntimePatchVersion) + $(MajorVersion).$(MinorVersion).$(SdkPatchVersion) + + + + 0.1.0 + alpha.1 + + + + 2.2.0 + + + + 0.1.0-6.0.100-bootstrap.3 + + diff --git a/src/SourceBuild/tarball/content/eng/install-nuget-credprovider.sh b/src/SourceBuild/tarball/content/eng/install-nuget-credprovider.sh new file mode 100644 index 000000000..880105e3c --- /dev/null +++ b/src/SourceBuild/tarball/content/eng/install-nuget-credprovider.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# This script installs the NuGet Credential Provider. It is intended for use on CI machines only. + +# Originally copied from https://github.com/dotnet/core-setup/blob/aa28510afc9b986c6837db6784d816fe4a66c7d0/eng/install-nuget-credprovider.sh + +set -e + +# Install curl if necessary. Dependency exists inside downloaded script. +if command -v curl > /dev/null; then + echo "curl found." +else + echo "curl not found, trying to install..." + ( + set +e + set -x + + apt update && apt install -y curl + + apk update && apk upgrade && apk add curl + + exit 0 + ) +fi + +# Install. Ported from https://gist.github.com/shubham90/ad85f2546a72caa20d57bce03ec3890f +install_credprovider() { + # Download the provider and install. + cred_provider_url='https://raw.githubusercontent.com/Microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh' + curl "$cred_provider_url" -s -S -L | bash + + # Environment variable to enable session token cache. More on this here: https://github.com/Microsoft/artifacts-credprovider#help + export NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED=true +} + +install_credprovider + +# Additional setup to try to avoid flakiness: https://github.com/dotnet/arcade/issues/3932 +export DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER=0 +export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20 +export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20 diff --git a/src/SourceBuild/tarball/content/global.json b/src/SourceBuild/tarball/content/global.json new file mode 100644 index 000000000..f48709aa6 --- /dev/null +++ b/src/SourceBuild/tarball/content/global.json @@ -0,0 +1,12 @@ +{ + "tools": { + "dotnet": "6.0.100-preview.5.21225.11" + }, + "msbuild-sdks": { + "Microsoft.Build.CentralPackageVersions": "2.0.1", + "Microsoft.Build.Traversal": "2.0.2", + "Microsoft.NET.Sdk.IL": "3.0.0-preview-27107-01", + "Microsoft.DotNet.Arcade.Sdk": "6.0.0-beta.21304.1", + "Yarn.MSBuild": "1.15.2" + } +} diff --git a/src/SourceBuild/tarball/content/keys/Newtonsoft.Json.snk b/src/SourceBuild/tarball/content/keys/Newtonsoft.Json.snk new file mode 100644 index 000000000..c26e9e4ea Binary files /dev/null and b/src/SourceBuild/tarball/content/keys/Newtonsoft.Json.snk differ diff --git a/src/SourceBuild/tarball/content/keys/NuGet.Client.snk b/src/SourceBuild/tarball/content/keys/NuGet.Client.snk new file mode 100644 index 000000000..695f1b387 Binary files /dev/null and b/src/SourceBuild/tarball/content/keys/NuGet.Client.snk differ diff --git a/src/SourceBuild/tarball/content/keys/README.md b/src/SourceBuild/tarball/content/keys/README.md new file mode 100644 index 000000000..d83cf9491 --- /dev/null +++ b/src/SourceBuild/tarball/content/keys/README.md @@ -0,0 +1,13 @@ +This directory contains the public key portion of keys used by different +projects so we can public sign (also known as OSS signing) assemblies that need +to be signed to have the correct strong name. + +These are used for projects which full sign their assemblies and don't have keys +checked in. + +To extract a key, take an existing built binary for a project (e.g. download a +nupkg from NuGet.org and then unpack one of the assemblies from it) and use `sn`: + +``` +sn -e +``` diff --git a/src/SourceBuild/tarball/content/prep.sh b/src/SourceBuild/tarball/content/prep.sh new file mode 100755 index 000000000..58cf90e40 --- /dev/null +++ b/src/SourceBuild/tarball/content/prep.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)" + +usage() { + echo "usage: $0" + echo "" + echo " Prepares a tarball to be built by downloading Private.SourceBuilt.Artifacts.*.tar.gz and" + echo " installing the version of dotnet referenced in global.json" + echo "" +} + +positional_args=() +while :; do + if [ $# -le 0 ]; then + break + fi + lowerI="$(echo "$1" | awk '{print tolower($0)}')" + case $lowerI in + "-?"|-h|--help) + usage + exit 0 + ;; + *) + positional_args+=("$1") + ;; + esac + + shift +done + +# Check for the archive text file which describes the location of the archive files to download +if [ ! -f $SCRIPT_ROOT/packages/archive/archiveArtifacts.txt ]; then + echo " ERROR: $SCRIPT_ROOT/packages/archive/archiveArtifacts.txt does not exist. Cannot determine which archives to download. Exiting..." + exit -1 +fi + +downloadArtifacts=true +installDotnet=true + +# Check to make sure curl exists to download the archive files +if ! command -v curl &> /dev/null +then + echo " ERROR: curl not found. Exiting..." + exit -1 +fi + +# Check if Private.SourceBuilt artifacts archive exists +if [ -f $SCRIPT_ROOT/packages/archive/Private.SourceBuilt.Artifacts.*.tar.gz ]; then + echo " Private.SourceBuilt.Artifacts.*.tar.gz exists...it will not be downloaded" + downloadArtifacts=false +fi + +# Check if dotnet is installed +if [ -d $SCRIPT_ROOT/.dotnet ]; then + echo " ./.dotnet SDK directory exists...it will not be installed" + installDotnet=false; +fi + +# Read the archive text file to get the archives to download and download them +while read -r line; do + if [[ $line == *"Private.SourceBuilt.Artifacts"* ]]; then + if [ "$downloadArtifacts" == "true" ]; then + echo " Downloading source-built artifacts..." + (cd $SCRIPT_ROOT/packages/archive/ && curl -O $line) + fi + fi +done < $SCRIPT_ROOT/packages/archive/archiveArtifacts.txt + +# Check for the version of dotnet to install +if [ "$installDotnet" == "true" ]; then + echo " Installing dotnet..." + (source ./eng/common/tools.sh && InitializeDotNetCli true) +fi diff --git a/src/SourceBuild/tarball/content/repos/Directory.Build.props b/src/SourceBuild/tarball/content/repos/Directory.Build.props new file mode 100644 index 000000000..851855392 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/Directory.Build.props @@ -0,0 +1,148 @@ + + + + $(MSBuildProjectName) + + + + + + + + + $(RepositoryName) + $(ClonedSubmoduleDirectory)$(SourceDirectory).$(GitCommitHash)/ + $(SubmoduleDirectory)$(SourceDirectory).$(GitCommitHash)/ + true + $(LoggingDir)$(RepositoryName).log + >> $(RepoConsoleLogFile) 2>&1 + true + + + $(CompletedSemaphorePath)$(RepositoryName)/ + + + + - + 0 + + + -- + false + + + + $(GitCommitDate.Replace('-', '')) + + + + + '$(RepositoryName)' + $(ProjectBuildReason) to produce tarball + $(ProjectBuildReason) in tarball + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(PackagesDir) + $(ArcadeBootstrapPackageDir) + $(ARCADE_BOOTSTRAP_VERSION) + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/Directory.Build.targets b/src/SourceBuild/tarball/content/repos/Directory.Build.targets new file mode 100644 index 000000000..91de0162c --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/Directory.Build.targets @@ -0,0 +1,756 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + true + $(ProjectDirectory)/eng/common/tools.sh + $(ProjectDirectory)/eng/common/build.sh + + + + $(RepoApiImplemented) + $(RepoApiImplemented) + $(RepoApiImplemented) + + + + $(ProjectDirectory)NuGet.config + $(ProjectDirectory)NuGet.Config + $(ProjectDirectory)src\NuGet.config + $(ProjectDirectory)src\NuGet.Config + + + + + $(RepoApiArgs) /p:DotNetPackageVersionPropsPath=$(PackageVersionPropsPath) + + + + $(RepoApiArgs) /p:DotNetRestoreSourcePropsPath=$(RestoreSourcePropsPath) + $(RepoApiArgs) /p:DotNetBuildOffline=true + + + + $(RepoApiArgs) /p:DotNetOutputBlobFeedDir=$(SourceBuiltBlobFeedDir) + + + + + + <_DependentProject Include="@(RepositoryReference -> '%(Identity).proj')" /> + + + + + + + + + + + + + git --work-tree=$(ProjectDirectory) apply --ignore-whitespace --whitespace=nowarn + + + + + + + + + + + + + + + + + + logger_path="%24toolset_dir"/%24%28cd "$toolset_dir" && find . -name Microsoft.DotNet.Arcade.Sdk.dll \( -regex '.*netcoreapp2.1.*' -or -regex '.*net5.0.*' \) ) + + + + logger_path="%24toolset_dir"/%24%28cd "$toolset_dir" && find . -name Microsoft.DotNet.ArcadeLogging.dll \( -regex '.*netcoreapp2.1.*' -or -regex '.*net5.0.*' \) ) + + + + $(ArcadeLoggingReplacementText) + if [[ ! -f $logger_path ]]; then + $(ArcadeSdkReplacementText) + fi + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_PossibleCliVersionJsonPath Include="sdk.version" /> + <_PossibleCliVersionJsonPath Include="tools.dotnet" /> + + + + + + + + + + <_PackageVersionPropsBackupPath>$(PackageVersionPropsPath).pre.$(RepositoryName).xml + + + + + <_AdditionalAssetDirs Include="$(SourceBuiltToolsetDir)" Condition="Exists('$(SourceBuiltToolsetDir)')" /> + + + + + + + + + + + + + + <_KnownOriginPackagePaths Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" /> + <_KnownOriginPackagePaths Include="$(PrebuiltPackagesPath)*.nupkg" /> + <_KnownOriginPackagePaths Include="$(ReferencePackagesDir)*.nupkg" /> + + + + + + + + + + + + <_ReportDir>$(ConflictingPackageReportDir)before-$(RepositoryName)/ + <_ReportDataFile>$(_ReportDir)usage.xml + <_ProjectAssetsJsonArchiveFile>$(_ReportDir)all-project-assets-json-files.zip + + + + + + + + + + + + + + + <_DotNetRestoreSources Include="$(ExtraRestoreSourcePath)" Condition="'$(ExtraRestoreSourcePath)' != ''"/> + <_DotNetRestoreSources Include="$(SourceBuiltPackagesPath)" /> + <_DotNetRestoreSources Include="$(ReferencePackagesDir)" Condition="'$(OfflineBuild)' == 'true'"/> + <_DotNetRestoreSources Include="$(PrebuiltPackagesPath)" Condition="'$(OfflineBuild)' == 'true'"/> + <_DotNetRestoreSources Include="$(PrebuiltSourceBuiltPackagesPath)" Condition="'$(OfflineBuild)' == 'true'"/> + + <_AdditionalAssetDirs Include="$(SourceBuiltToolsetDir)" Condition="Exists('$(SourceBuiltToolsetDir)')" /> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + $(BuildCommand) /v:$(LogVerbosity) $(RepoApiArgs) $(RedirectRepoOutputToLog) + $(BuildCommand) $(RepoApiArgs) $(RedirectRepoOutputToLog) + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_BuiltPackages Condition="'$(PackagesOutput)' != ''" Include="$(PackagesOutput)/*.nupkg" Exclude="$(PackagesOutput)/*.symbols.nupkg"/> + <_BuiltPackages Condition="'@(PackagesOutputList)' != ''" Include="%(PackagesOutputList.Identity)/*.nupkg" Exclude="%(PackagesOutputList.Identity)/*.symbols.nupkg"/> + + + + + + <_BuiltIntermediatePackages Condition="'$(PackagesOutput)' != ''" Include="$(PackagesOutput)/Microsoft.SourceBuild.Intermediate.*.nupkg" Exclude="$(PackagesOutput)/*.symbols.nupkg"/> + <_BuiltIntermediatePackages Condition="'@(PackagesOutputList)' != ''" Include="%(PackagesOutputList.Identity)/Microsoft.SourceBuild.Intermediate.*.nupkg" Exclude="%(PackagesOutputList.Identity)/*.symbols.nupkg"/> + + + + + <_DestinationPath>$(SourceBuiltPackagesPath) + + <_DestinationPath Condition="$([System.String]::Copy(%(_BuiltIntermediatePackages.Identity)).Contains('source-build-reference-packages'))">$(ReferencePackagesDir) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_FilesToCopy Include="$(PackagesDir)$([System.String]::copy('%(_BuiltPackageInfos.PackageId)').ToLower())/%(_BuiltPackageInfos.PackageVersion)/**/*.nupkg" /> + <_FilesToDelete Include="$(PackagesDir)$([System.String]::copy('%(_BuiltPackageInfos.PackageId)').ToLower())/%(_BuiltPackageInfos.PackageVersion)/**/*.*" /> + + + + + + + + + + <_ToolPackage + Condition="'%(BuiltSdkPackageOverride.Version)' == ''" + Include="$(SourceBuiltPackagesPath)%(BuiltSdkPackageOverride.Identity)*.nupkg" + Exclude="$(SourceBuiltPackagesPath)%(BuiltSdkPackageOverride.Identity)*.symbols.nupkg" + Id="%(BuiltSdkPackageOverride.Identity)" /> + <_ToolPackage + Condition="'%(BuiltSdkPackageOverride.Version)' != ''" + Include="$(SourceBuiltPackagesPath)%(BuiltSdkPackageOverride.Identity).%(BuiltSdkPackageOverride.Version).nupkg" + Exclude="$(SourceBuiltPackagesPath)%(BuiltSdkPackageOverride.Identity).%(BuiltSdkPackageOverride.Version).symbols.nupkg" + Id="%(BuiltSdkPackageOverride.Identity)" /> + + + + + + + + + + + + + + + + + + + + + + + + <_PackagesNotCreatedReason Include="^ There may have been a silent failure in the submodule build. To confirm, check the build log file for undetected errors that may have prevented package creation: $(RepoConsoleLogFile)" /> + <_PackagesNotCreatedReason Include="^ This error might be a false positive if $(RepositoryName) intentionally builds no nuget packages. If so, set the SkipEnsurePackagesCreated property to true in $(MSBuildProjectFullPath)" /> + <_PackagesNotCreatedReason Include="^ The 'bin' directory might be dirty from a previous build and the package files already existed. If so, perform a clean build, or check which packages were already in 'bin' by opening $(_PackageVersionPropsBackupPath)" /> + <_PackagesNotCreatedReason Include="^ The packages may have been written to an unexpected directory. For example, some repos used bin/ and changed to artifacts/ to match Arcade. Check PackagesOutput in $(MSBuildProjectFullPath) (currently '$(PackagesOutput)')" /> + + + + + + + + + + + + + + + + + + + + + $(LocalNuGetPackagesRoot)$(RepositoryName)/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + false + + + + + + + + + + false + true + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/application-insights.proj b/src/SourceBuild/tarball/content/repos/application-insights.proj new file mode 100644 index 000000000..f62473e51 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/application-insights.proj @@ -0,0 +1,44 @@ + + + + ApplicationInsights-dotnet + + + + + + $(ProjectDirectory)/bin/$(Configuration) + false + true + + + + + + + $(ProjectDirectory)/Microsoft.ApplicationInsights.csproj + $(BuildCommandArgs) /p:Configuration=$(Configuration) + + $(BuildCommandArgs) /p:EnlistmentRoot=$(ProjectDirectory)/src + $(BuildCommandArgs) /p:RelativeOutputPathBase= + $(BuildCommandArgs) /v:$(LogVerbosity) + $(BuildCommandArgs) $(RedirectRepoOutputToLog) + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/arcade.proj b/src/SourceBuild/tarball/content/repos/arcade.proj new file mode 100644 index 000000000..719b3e50d --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/arcade.proj @@ -0,0 +1,78 @@ + + + + + + + $(BuildCommandArgs) $(FlagParameterPrefix)pack + $(BuildCommandArgs) $(FlagParameterPrefix)configuration $(Configuration) + $(BuildCommandArgs) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg) + + true + $(BuildCommandArgs) -v $(LogVerbosity) + $(BuildCommandArgs) -bl + $(BuildCommandArgs) -ci + $(BuildCommandArgs) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg) + $(BuildCommandArgs) /p:ArcadeBuildFromSource=true + $(BuildCommandArgs) /p:CopyWipIntoInnerSourceBuildRepo=true + + $(ProjectDirectory)\build$(ShellExtension) $(BuildCommandArgs) + + + $(ProjectDirectory)/NuGet.config + + $(ProjectDirectory)global.json + + true + false + + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + + + true + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + %(PreviouslySourceBuiltNuGetPackageInfo.PackageVersion) + $(BuildCommand) /p:NuGetVersion=$(PreviouslySourceBuiltNuGetVersion) + + + + + diff --git a/src/SourceBuild/tarball/content/repos/aspnet-xdt.proj b/src/SourceBuild/tarball/content/repos/aspnet-xdt.proj new file mode 100644 index 000000000..c51910bad --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/aspnet-xdt.proj @@ -0,0 +1,33 @@ + + + + xdt + + + + + + + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) --binaryLog + $(BuildCommandArgs) -ci + $(BuildCommandArgs) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg) + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)global.json + $(ProjectDirectory)/NuGet.config + true + false + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/aspnetcore.proj b/src/SourceBuild/tarball/content/repos/aspnetcore.proj new file mode 100644 index 000000000..50e459d14 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/aspnetcore.proj @@ -0,0 +1,93 @@ + + + aspnetcore + + + + + + $(TargetRid) + freebsd-x64 + osx-x64 + + $(BuildCommandArgs) --restore --build --pack + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) --ci + $(BuildCommandArgs) -bl + $(BuildCommandArgs) /v:$(LogVerbosity) + + $(BuildCommandArgs) --arch $(Platform) + $(BuildCommandArgs) /p:BuildNodeJs=false + $(BuildCommandArgs) /p:SourceBuildRuntimeIdentifier=$(OverrideTargetRid) + $(BuildCommandArgs) /p:UseAppHost=false + $(BuildCommandArgs) /p:PublishCompressedFilesPathPrefix=$(SourceBuiltAspNetCoreRuntime) + + + $(BuildCommandArgs) /p:MicrosoftNetFrameworkReferenceAssembliesVersion=1.0.0 + + true + + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + false + true + + $(ProjectDirectory)global.json + $(ProjectDirectory)NuGet.config + + $(SourceBuiltPackagesPath) + $(EnvironmentRestoreSources)%3B$(ReferencePackagesDir)%3B$(PrebuiltPackagesPath) + $(EnvironmentRestoreSources)%3Bhttps://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public%40Local/nuget/v3/index.json + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/clicommandlineparser.proj b/src/SourceBuild/tarball/content/repos/clicommandlineparser.proj new file mode 100644 index 000000000..19a01f0a9 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/clicommandlineparser.proj @@ -0,0 +1,41 @@ + + + + CliCommandLineParser + + + + + + + $(FlagParameterPrefix)pack $(FlagParameterPrefix)configuration $(Configuration) + $(BuildCommandArgs) --projects $(ProjectDirectory)CommandLine.sln + $(BuildCommandArgs) $(FlagParameterPrefix)restore + + + true + $(BuildCommandArgs) -v $(LogVerbosity) + $(BuildCommandArgs) -bl + $(BuildCommandArgs) -ci + + $(BuildCommandArgs) /p:DotNetPackageVersionPropsPath=$(PackageVersionPropsPath) + + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + false + $(ProjectDirectory)global.json + + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + false + $(ProjectDirectory)/NuGet.config + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/command-line-api.proj b/src/SourceBuild/tarball/content/repos/command-line-api.proj new file mode 100644 index 000000000..57e458792 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/command-line-api.proj @@ -0,0 +1,30 @@ + + + + + + --restore --build --pack + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) --binaryLog + $(BuildCommandArgs) -ci + $(BuildCommandArgs) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg) + $(BuildCommandArgs) /p:Projects=$(ProjectDirectory)source-build.slnf + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)global.json + $(ProjectDirectory)NuGet.config + true + false + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/common.proj b/src/SourceBuild/tarball/content/repos/common.proj new file mode 100644 index 000000000..2c04a0bfa --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/common.proj @@ -0,0 +1,24 @@ + + + + + aspnet + servicing/1.0.x + $(ProjectDirectory)artifacts/build/ + /p:Configuration=$(Configuration) + $(ProjectDirectory)/build$(ShellExtension) -NoTest $(BuildArguments) + $(ProjectDirectory)/clean$(ShellExtension) $(BuildArguments) + false + true + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/cssparser.proj b/src/SourceBuild/tarball/content/repos/cssparser.proj new file mode 100644 index 000000000..2bd2e2608 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/cssparser.proj @@ -0,0 +1,37 @@ + + + + + + $(ProjectDirectory)/src/Microsoft.Css.Parser/bin/$(Configuration)/ + false + true + + + + + + + $(ProjectDirectory)/src/Microsoft.Css.Parser/Microsoft.Css.Parser.csproj + $(BuildCommandArgs) /p:Configuration=$(Configuration) + $(BuildCommandArgs) /v:$(LogVerbosity) + $(BuildCommandArgs) $(RedirectRepoOutputToLog) + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/diagnostics.proj b/src/SourceBuild/tarball/content/repos/diagnostics.proj new file mode 100644 index 000000000..169fe0965 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/diagnostics.proj @@ -0,0 +1,43 @@ + + + + + + $(ProjectDirectory)global.json + $(ProjectDirectory)NuGet.config + $(ProjectDirectory)/artifacts/packages/$(Configuration)/Shipping + false + true + + + + + + + + + + + $(ProjectDirectory)/src/Microsoft.Diagnostics.NETCore.Client/Microsoft.Diagnostics.NETCore.Client.csproj + $(BuildCommandArgs) /p:Configuration=$(Configuration) + $(BuildCommandArgs) /v:$(LogVerbosity) + $(BuildCommandArgs) $(RedirectRepoOutputToLog) + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/fsharp.proj b/src/SourceBuild/tarball/content/repos/fsharp.proj new file mode 100644 index 000000000..22dd1b7e7 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/fsharp.proj @@ -0,0 +1,39 @@ + + + + + --restore --build --pack --publish + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) --binaryLog + $(BuildCommandArgs) --ci + $(BuildCommandArgs) /p:FSharpSourceBuild=true + $(BuildCommandArgs) /p:DotNetBuildFromSource=true + $(BuildCommandArgs) /p:TreatWarningsAsErrors=false + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + false + $(ProjectDirectory)global.json + $(ProjectDirectory)/NuGet.config + true + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/humanizer.proj b/src/SourceBuild/tarball/content/repos/humanizer.proj new file mode 100644 index 000000000..a4cb2a6b2 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/humanizer.proj @@ -0,0 +1,48 @@ + + + + Humanizer + + + + + + $(ProjectDirectory)/src/Humanizer/bin/$(Configuration) + false + true + $(ProjectDirectory)/src/NuGet.config + + + + + + + + + + + $(ProjectDirectory)src/Humanizer/Humanizer.csproj + $(BuildCommandArgs) /p:Configuration=$(Configuration) + $(BuildCommandArgs) /v:$(LogVerbosity) + $(BuildCommandArgs) $(RedirectRepoOutputToLog) + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/installer.proj b/src/SourceBuild/tarball/content/repos/installer.proj new file mode 100644 index 000000000..fd8a95be0 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/installer.proj @@ -0,0 +1,135 @@ + + + installer + + + + + + $(TargetRid) + osx-x64 + $(OverrideTargetRid.Substring(0, $(OverrideTargetRid.IndexOf("-")))) + + --runtime-id $(OverrideTargetRid) + --runtime-id $(TargetRid) + + --restore --build --pack --ci --binaryLog + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) $(RuntimeArg) + + + $(BuildCommandArgs) /p:NETCoreAppMaximumVersion=99.9 + $(BuildCommandArgs) /p:OSName=$(OSNameOverride) + $(BuildCommandArgs) /p:Rid=$(TargetRid) + $(BuildCommandArgs) /p:DOTNET_INSTALL_DIR=$(DotNetCliToolDir) + + + $(BuildCommandArgs) /p:AspNetCoreSharedFxInstallerRid=linux-$(Platform) + + $(BuildCommandArgs) /p:CoreSetupRid=freebsd-x64 /p:PortableBuild=true + $(BuildCommandArgs) /p:CoreSetupRid=osx-x64 + $(BuildCommandArgs) /p:CoreSetupRid=$(TargetRid) + + + $(BuildCommandArgs) /p:CoreSetupBlobRootUrl=file:%2F%2F$(LocalBlobStorageRoot) + + + $(BuildCommandArgs) /p:DotnetToolsetBlobRootUrl=file:%2F%2F$(LocalBlobStorageRoot) + + $(BuildCommandArgs) /p:SkipBuildingInstallers=true + $(BuildCommandArgs) /p:IncludeNuGetPackageArchive=false + $(BuildCommandArgs) /p:IncludeAdditionalSharedFrameworks=false + $(BuildCommandArgs) /p:UsePortableLinuxSharedFramework=false + $(BuildCommandArgs) /p:IncludeSharedFrameworksForBackwardsCompatibilityTests=false + + $(ProjectDirectory)/build$(ShellExtension) $(BuildCommandArgs) + + win-x64 + osx-x64 + freebsd-x64 + $(TargetRid) + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + $(PackagesOutput) + false + true + true + + $(SourceBuiltPackagesPath) + $(EnvironmentExternalRestoreSources)%3B$(ReferencePackagesDir)%3B$(PrebuiltPackagesPath) + + + true + $(ProjectDirectory)/NuGet.config + + + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/known-good-tests.proj b/src/SourceBuild/tarball/content/repos/known-good-tests.proj new file mode 100644 index 000000000..e8d75dd4b --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/known-good-tests.proj @@ -0,0 +1,37 @@ + + + + + false + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/known-good.proj b/src/SourceBuild/tarball/content/repos/known-good.proj new file mode 100644 index 000000000..6d7f42094 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/known-good.proj @@ -0,0 +1,43 @@ + + + + + false + true + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/linker.proj b/src/SourceBuild/tarball/content/repos/linker.proj new file mode 100644 index 000000000..823dde083 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/linker.proj @@ -0,0 +1,81 @@ + + + + + + Microsoft.NET.ILLink.Tasks + + $(BuildCommandArgs) $(FlagParameterPrefix)ci + $(BuildCommandArgs) $(FlagParameterPrefix)configuration $(Configuration) + $(BuildCommandArgs) $(FlagParameterPrefix)restore + $(BuildCommandArgs) $(FlagParameterPrefix)build + $(BuildCommandArgs) $(FlagParameterPrefix)pack + $(BuildCommandArgs) $(FlagParameterPrefix)publish + $(BuildCommandArgs) -bl + + $(BuildCommandArgs) /p:ArcadeBuildFromSource=true + $(BuildCommandArgs) /p:CopyWipIntoInnerSourceBuildRepo=true + + $(ProjectDirectory)\build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + + false + $(ProjectDirectory)/NuGet.config + true + + + + + + + + + + + + + + $(ProjectDirectory)src/ILLink.Tasks/ILLink.Tasks.nuspec + + + + + $id$ + $version$ + $authors$ + $description$ + + + + + + + +]]> + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/msbuild.proj b/src/SourceBuild/tarball/content/repos/msbuild.proj new file mode 100644 index 000000000..d635daa70 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/msbuild.proj @@ -0,0 +1,105 @@ + + + + + $(ProjectDirectory)/artifacts/packages/$(Configuration)/Shipping + $(ProjectDirectory)/artifacts/packages/$(Configuration)/NonShipping + + $(OutputVersionArgs) /p:DisableNerdbankVersioning=true + + --build --publish --pack + $(BuildCommandArgs) /p:DotNetBuildFromSource=true + $(BuildCommandArgs) /p:DotNetCoreSdkDir=$(DotNetCliToolDir) + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) --projects $(ProjectDirectory)MSBuild.SourceBuild.slnf + $(BuildCommandArgs) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg) + $(BuildCommandArgs) $(OutputVersionArgs) + $(BuildCommandArgs) /p:DotNetPackageVersionPropsPath=$(PackageVersionPropsPath) + + + $(BuildCommandArgs) --warnAsError false + + true + $(BuildCommandArgs) -v $(LogVerbosity) + $(BuildCommandArgs) -bl + $(BuildCommandArgs) -ci + + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + + $(ProjectDirectory)/NuGet.config + + true + false + + $(ProjectDirectory)global.json + + $(SourceBuiltPackagesPath) + $(EnvironmentRestoreSources)%3B$(ReferencePackagesDir)%3B$(PrebuiltPackagesPath) + $(EnvironmentRestoreSources)%3Bhttps://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public%40Local/nuget/v3/index.json + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + <_CentralVersionsToolPackage + Include="$(ReferencePackagesDir)%(CentralPackageVersionsSdkOverride.Identity)*.nupkg" + Id="%(CentralPackageVersionsSdkOverride.Identity)" /> + + + + $(ToolPackageExtractDir)%(_CentralVersionsToolPackage.Id)/ + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/netcorecli-fsc.proj b/src/SourceBuild/tarball/content/repos/netcorecli-fsc.proj new file mode 100644 index 000000000..a9c1d27d3 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/netcorecli-fsc.proj @@ -0,0 +1,20 @@ + + + + + /bl /v:$(LogVerbosity) /p:OutputPath=$(OutputPath)$(RepositoryName)/ /p:BaseIntermediateOutputPath=$(IntermediatePath)$(RepositoryName) + $(DotnetToolCommand) pack -c $(Configuration) --output $(SourceBuiltPackagesPath) --no-build FSharp.NET.Sdk.csproj /p:NuspecFile=FSharp.NET.Sdk.nuspec $(OutputArgs) $(RedirectRepoOutputToLog) + $(SourceBuiltPackagesPath) + false + $(SubmoduleDirectory)$(RepositoryName)/ + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/newtonsoft-json.proj b/src/SourceBuild/tarball/content/repos/newtonsoft-json.proj new file mode 100644 index 000000000..1bebd52f0 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/newtonsoft-json.proj @@ -0,0 +1,39 @@ + + + + Newtonsoft.Json + + + + + + + + + + + + + + $(ProjectDirectory)/Src/NuGet.Config + $(KeysDir)Newtonsoft.Json.snk + $(ProjectDirectory)/Src/Newtonsoft.Json/ + $(NewtonsoftJsonDirectory)Newtonsoft.Json.csproj + "/p:DotnetOnly=true" "/p:Configuration=$(Configuration)" "/p:AssemblyOriginatorKeyFile=$(NewtonsoftJsonKeyFilePath)" "/p:SignAssembly=true" "/p:PublicSign=true" "/p:TreatWarningsAsErrors=false" "/p:AdditionalConstants=SIGNED" + $(DotnetToolCommand) build $(NewtonsoftJsonProjectPath) /bl:build.binlog $(DotnetToolCommandArguments) + $(DotnetToolCommand) pack $(NewtonsoftJsonProjectPath) /bl:pack.binlog $(DotnetToolCommandArguments) + $(DotnetToolCommand) clean $(NewtonsoftJsonProjectPath) $(DotnetToolCommandArguments) + $(NewtonsoftJsonDirectory)bin/$(Configuration)/ + false + true + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/newtonsoft-json901.proj b/src/SourceBuild/tarball/content/repos/newtonsoft-json901.proj new file mode 100644 index 000000000..c87ec499a --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/newtonsoft-json901.proj @@ -0,0 +1,31 @@ + + + + Newtonsoft.Json + + + + + + $(ProjectDirectory)/Src/NuGet.Config + $(KeysDir)Newtonsoft.Json.snk + $(ProjectDirectory)/Src/Newtonsoft.Json/ + $(NewtonsoftJsonDirectory)Newtonsoft.Json.Dotnet.csproj + "/p:DotnetOnly=true" "/p:Configuration=$(Configuration)" "/p:AssemblyOriginatorKeyFile=$(NewtonsoftJsonKeyFilePath)" "/p:SignAssembly=true" "/p:PublicSign=true" "/p:TreatWarningsAsErrors=false" "/p:AdditionalConstants=SIGNED" + $(DotnetToolCommand) build $(NewtonsoftJsonProjectPath) $(DotnetToolCommandArguments) + $(DotnetToolCommand) pack $(NewtonsoftJsonProjectPath) $(DotnetToolCommandArguments) + $(DotnetToolCommand) clean $(NewtonsoftJsonProjectPath) $(DotnetToolCommandArguments) + $(NewtonsoftJsonDirectory)bin/$(Configuration)/ + false + true + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/nuget-client.proj b/src/SourceBuild/tarball/content/repos/nuget-client.proj new file mode 100644 index 000000000..326263038 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/nuget-client.proj @@ -0,0 +1,68 @@ + + + + nuget.client + + + + + + $(ProjectDirectory)artifacts/nupkgs/ + $(ProjectDirectory)src/NuGet.Core/ + $(ProjectDirectory)cli/ + $(KeysDir)NuGet.Client.snk + false + true + true + + + + + + + + + + + + + + + + $(DotnetToolCommand) msbuild $(ProjectDirectory)/build/build.proj + $(BuildCommandBase) /p:VisualStudioVerion=15.0 + $(BuildCommandBase) /p:Configuration=$(Configuration) + $(BuildCommandBase) /p:BuildRTM=false + $(BuildCommandBase) /p:BuildNumber=$(OfficialBuildId) + $(BuildCommandBase) /v:$(LogVerbosity) + $(BuildCommandBase) /p:TreatWarningsAsErrors=false + $(BuildCommandBase) /p:DotNetPackageVersionPropsPath=$(PackageVersionPropsPath) + + + + + + + + $(BuildCommandBase) /t:PackXPlat + $(PackCommand) /p:PackageOutputPath=$(PackagesOutput) + $(PackCommand) /p:NoPackageAnalysis=true + $(PackCommand) /flp:v=detailed + $(PackCommand) /v:$(LogVerbosity) + $(PackCommand) /bl:pack.binlog + $(PackCommand) $(RedirectRepoOutputToLog) + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/package-source-build.proj b/src/SourceBuild/tarball/content/repos/package-source-build.proj new file mode 100644 index 000000000..d5f3a16b6 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/package-source-build.proj @@ -0,0 +1,38 @@ + + + + + false + $(SubmoduleDirectory)$(RepositoryName)/ + true + $(PackageVersionPropsPath) + $(GennedPackageVersionPropsPath) + + + + + + + + + + + + + + + + + + + + + $(OutputPath)$(SourceBuiltArtifactsTarballName).$(VersionPrefix)-$(VersionSuffix).tar.gz + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/roslyn-analyzers.proj b/src/SourceBuild/tarball/content/repos/roslyn-analyzers.proj new file mode 100644 index 000000000..3d517f6ef --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/roslyn-analyzers.proj @@ -0,0 +1,38 @@ + + + + + --restore --build --pack + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) -v $(LogVerbosity) + $(BuildCommandArgs) -bl + $(BuildCommandArgs) -warnaserror false + $(BuildCommandArgs) /p:TreatWarningsAsErrors=false + + + $(BuildCommandArgs) /p:OfficialBuild=true + $(BuildCommandArgs) /p:DotNetPackageVersionPropsPath=$(PackageVersionPropsPath) + $(BuildCommandArgs) /p:DotNetBuildFromSource=true + $(BuildCommandArgs) /p:GitHubRepositoryName=roslyn-analyzers + $(BuildCommandArgs) /p:RepositoryUrl=git://github.com/dotnet/roslyn-analyzers + + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + false + true + true + $(ProjectDirectory)global.json + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/roslyn.proj b/src/SourceBuild/tarball/content/repos/roslyn.proj new file mode 100644 index 000000000..4b992ed3c --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/roslyn.proj @@ -0,0 +1,78 @@ + + + + + + + --restore --build --pack + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) -v $(LogVerbosity) + $(BuildCommandArgs) -bl + $(BuildCommandArgs) /p:TreatWarningsAsErrors=false + + + $(BuildCommandArgs) /p:OfficialBuild=true + $(BuildCommandArgs) /p:ApplyPartialNgenOptimization=false + $(BuildCommandArgs) /p:EnablePartialNgenOptimization=false + $(BuildCommandArgs) /p:PublishWindowsPdb=false + $(BuildCommandArgs) /p:DotNetPackageVersionPropsPath=$(PackageVersionPropsPath) + $(BuildCommandArgs) /p:DotNetBuildFromSource=true + + + $(BuildCommandArgs) /p:UsingToolMicrosoftNetCompilers=false + + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + false + true + $(ProjectDirectory)global.json + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/runtime-portable.proj b/src/SourceBuild/tarball/content/repos/runtime-portable.proj new file mode 100644 index 000000000..0faeb9afc --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/runtime-portable.proj @@ -0,0 +1,56 @@ + + + + runtime + true + false + + + + + + + + + + + + + + + + + + $(ProjectDirectory)artifacts/portableLog + $(ProjectDirectory)artifacts/portableObj + + + + + + + + + + + + + + + + + + + + + + + + + + <_BuiltPackages Include="$(ShippingPackagesOutput)/*linux*nupkg" Exclude="$(ShippingPackagesOutput)/*.symbols.nupkg" /> + <_BuiltPackages Include="$(NonShippingPackagesOutput)/*linux*nupkg" Exclude="$(NonShippingPackagesOutput)/*.symbols.nupkg" /> + + + + diff --git a/src/SourceBuild/tarball/content/repos/runtime.common.props b/src/SourceBuild/tarball/content/repos/runtime.common.props new file mode 100644 index 000000000..16e04fcd5 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/runtime.common.props @@ -0,0 +1,143 @@ + + + runtime + + + + + + + + $(PortableBuild) + true + + $(TargetRid) + osx-x64 + freebsd-x64 + win-x64 + linux-$(Platform) + + false + false + $(UseSystemLibraries) + + + + + + $(ProjectDirectory)global.json + true + + + $(CoreClrBuildArguments) -cmakeargs -DCLR_CMAKE_USE_SYSTEM_LIBUNWIND=TRUE + $(CoreClrBuildArguments) skipnuget cross -skiprestore cmakeargs -DFEATURE_GDBJIT=TRUE + $(CoreClrBuildArguments) -clang6.0 /p:PortableBuild=true + + + $(CoreClrBuildArguments) /p:PortableBuild=false + + $(CoreClrBuildArguments) /p:PackagesDir=$(PackagesDir) + $(CoreClrBuildArguments) /p:ContinuousIntegrationBuild=true + $(CoreClrBuildArguments) /p:PackageRid=$(OverrideTargetRid) + + $(CoreClrBuildArguments) /p:NoPgoOptimize=true + $(CoreClrBuildArguments) /p:KeepNativeSymbols=true + $(CoreClrBuildArguments) /p:RuntimeOS=$(OverrideTargetRid.Substring(0, $(OverrideTargetRid.IndexOf("-")))) + $(CoreClrBuildArguments) /p:RuntimeOS=$(OverrideTargetRid) + + + + $(LibrariesBuildArguments) $(FlagParameterPrefix)restore $(FlagParameterPrefix)build $(FlagParameterPrefix)pack /p:SkipTests=true + $(LibrariesBuildArguments) $(FlagParameterPrefix)restore $(FlagParameterPrefix)build /p:IncludeTestUtils=true + $(LibrariesBuildArguments) $(FlagParameterPrefix)restore $(FlagParameterPrefix)buildtests $(FlagParameterPrefix)test /p:IncludeTestUtils=true + $(LibrariesBuildArguments) /p:TestRspFile=$(TestExclusionsDir)corefx/linux.docker.rsp + $(LibrariesBuildArguments) /p:ConfigurationGroup=$(Configuration) + $(LibrariesBuildArguments) /p:PackageRid=$(OverrideTargetRid) + + $(LibrariesBuildArguments) /p:RuntimeOS=$(OverrideTargetRid.Substring(0, $(OverrideTargetRid.IndexOf("-")))) + $(LibrariesBuildArguments) /p:RuntimeOS=$(OverrideTargetRid) + $(LibrariesBuildArguments) /p:PortableBuild=$(OverridePortableBuild) + $(LibrariesBuildArguments) /p:BuildAllPackages=true + $(LibrariesBuildArguments) /p:BuildAllOOBPackages=true + $(LibrariesBuildArguments) /p:KeepNativeSymbols=true + $(LibrariesBuildArguments) /p:BuiltSdkPackageOverride="" + $(LibrariesBuildArguments) /p:MicrosoftNETCoreDotNetHostPackageVersion=$(DOTNET_HOST_BOOTSTRAP_VERSION) + $(LibrariesBuildArguments) /p:MicrosoftNETCoreDotNetHostPolicyPackageVersion=$(DOTNET_HOST_BOOTSTRAP_VERSION) + + + + + $(FlagParameterPrefix)restore $(FlagParameterPrefix)build + $(InstallerBuildArguments) /p:PortableBuild=$(OverridePortableBuild) + $(InstallerBuildArguments) /p:KeepNativeSymbols=true + $(InstallerBuildArguments) -cmakeargs -DCLR_CMAKE_USE_SYSTEM_LIBUNWIND=TRUE + $(InstallerBuildArguments) /p:TargetArchitecture=$(Platform) /p:DisableCrossgen=true /p:CrossBuild=true + $(InstallerBuildArguments) /p:BuildDebPackage=false + $(InstallerBuildArguments) /p:BuildAllPackages=true + $(InstallerBuildArguments) /p:RestoreAllBuildRids=false + $(InstallerBuildArguments) /p:OutputRid=$(OverrideTargetRid) + $(InstallerBuildArguments) /p:DotNetOutputBlobFeedDir=$(SourceBuiltBlobFeedDir) + $(InstallerBuildArguments) /p:PublishCompressedFilesPathPrefix=$(SourceBuiltRuntimeDir) + $(InstallerBuildArguments) /p:BuiltSdkPackageOverride="" + $(InstallerBuildArguments) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg) + + $(FlagParameterPrefix)arch $(Platform) + $(CommonBuildArguments) $(FlagParameterPrefix)configuration $(Configuration) + $(CommonBuildArguments) /p:RuntimeConfiguration=$(Configuration) + $(CommonBuildArguments) /p:MicrosoftNetFrameworkReferenceAssembliesVersion=1.0.0 + $(CommonBuildArguments) $(FlagParameterPrefix)binaryLog + $(CommonBuildArguments) $(FlagParameterPrefix)ci + $(CommonBuildArguments) $(FlagParameterPrefix)runtimeConfiguration $(Configuration) + $(CommonBuildArguments) $(FlagParameterPrefix)verbosity $(LogVerbosity) + + $(CommonBuildArguments) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg) + $(CommonBuildArguments) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg) + + $(ProjectDirectory)build-source-build$(ShellExtension) $(CommonBuildArguments) $(FlagParameterPrefix)coreclr-args $(CoreClrBuildArguments) $(FlagParameterPrefix)libraries-args $(LibrariesBuildArguments) $(FlagParameterPrefix)installer-args $(InstallerBuildArguments) $(FlagParameterPrefix)additional-args + $(ArmEnvironmentVariables) $(BuildCommand) + $(ProjectDirectory)/clean$(ShellExtension) + + + + + false + $(ProjectDirectory)bin/Product/$(TargetOS).$(Platform).$(Configuration)/.nuget/pkg + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + $(ProjectDirectory)/NuGet.config + true + + + + + $(SourceBuiltPackagesPath) + $(EnvironmentRestoreSources)%3B$(ShippingPackagesOutput) + $(EnvironmentRestoreSources)%3B$(NonShippingPackagesOutput) + $(EnvironmentRestoreSources)%3B$(ReferencePackagesDir)%3B$(PrebuiltPackagesPath) + $(EnvironmentRestoreSources)%3Bhttps://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public%40Local/nuget/v3/index.json + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/runtime.common.targets b/src/SourceBuild/tarball/content/repos/runtime.common.targets new file mode 100644 index 000000000..a6d5f88ce --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/runtime.common.targets @@ -0,0 +1,72 @@ + + + + + + + + + + + + + + + $(ProjectDirectory)pkg/Microsoft.NETCore.Platforms/runtime.json + + + + + + + + + $(ToolPackageExtractDir)coreclr-tools + so + so + dylib + please define AssemblyExtension for $(TargetOS) + + + + + + + + + + + + + + + + + + + <_builtRuntimePackages Include="$(SourceBuiltAssetsDir)*.symbols.nupkg" /> + <_builtRuntimePackages> + $([System.String]::Copy('%(FileName)').Replace('symbols', 'nupkg')) + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/runtime.proj b/src/SourceBuild/tarball/content/repos/runtime.proj new file mode 100644 index 000000000..f20c04e17 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/runtime.proj @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/sdk.proj b/src/SourceBuild/tarball/content/repos/sdk.proj new file mode 100644 index 000000000..e07149dab --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/sdk.proj @@ -0,0 +1,66 @@ + + + sdk + + + + + + --pack --configuration $(Configuration) + $(BuildCommandArgs) /p:PackageProjectUrl=https://github.com/dotnet/sdk + $(BuildCommandArgs) /p:PublishCompressedFilesPathPrefix=$(SourceBuiltToolsetDir) + + true + $(BuildCommandArgs) -v $(LogVerbosity) + $(BuildCommandArgs) -bl + $(BuildCommandArgs) -ci + $(BuildCommandArgs) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg) + $(BuildCommandArgs) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg) + $(BuildCommandArgs) /p:Projects=$(ProjectDirectory)source-build.slnf + + + $(BuildCommandArgs) /p:ProduceReferenceAssembly=false + + + $(BuildCommandArgs) /p:PB_PackageVersionPropsUrl=file:%2F%2F$(PackageVersionPropsPath) + + $(ProjectDirectory)\build$(ShellExtension) $(BuildCommandArgs) + + + $(ProjectDirectory)/NuGet.config + $(ProjectDirectory)global.json + + true + false + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + + $(SourceBuiltPackagesPath) + $(EnvironmentRestoreSources)%3B$(ReferencePackagesDir)%3B$(PrebuiltPackagesPath) + $(EnvironmentRestoreSources)%3Bhttps://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public%40Local/nuget/v3/index.json + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/source-build-reference-packages.proj b/src/SourceBuild/tarball/content/repos/source-build-reference-packages.proj new file mode 100644 index 000000000..0bfe4116c --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/source-build-reference-packages.proj @@ -0,0 +1,30 @@ + + + + + $(BuildCommandArgs) $(FlagParameterPrefix)ci + $(BuildCommandArgs) $(FlagParameterPrefix)configuration $(Configuration) + $(BuildCommandArgs) $(FlagParameterPrefix)restore + $(BuildCommandArgs) $(FlagParameterPrefix)build + $(BuildCommandArgs) $(FlagParameterPrefix)pack + $(BuildCommandArgs) $(FlagParameterPrefix)publish + $(BuildCommandArgs) -bl + + $(BuildCommandArgs) /p:ArcadeBuildFromSource=true + $(BuildCommandArgs) /p:CopyWipIntoInnerSourceBuildRepo=true + + $(ProjectDirectory)\build$(ShellExtension) $(BuildCommandArgs) + + false + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + + + true + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/sourcelink.proj b/src/SourceBuild/tarball/content/repos/sourcelink.proj new file mode 100644 index 000000000..e0b4cca88 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/sourcelink.proj @@ -0,0 +1,34 @@ + + + + + + $(BuildCommandArgs) $(FlagParameterPrefix)pack + $(BuildCommandArgs) $(FlagParameterPrefix)configuration $(Configuration) + $(BuildCommandArgs) $(FlagParameterPrefix)binaryLog + $(BuildCommandArgs) $(FlagParameterPrefix)ci + $(BuildCommandArgs) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg) + $(BuildCommandArgs) /p:ArcadeBuildFromSource=true + $(BuildCommandArgs) /p:CopyWipIntoInnerSourceBuildRepo=true + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)global.json + $(ProjectDirectory)NuGet.config + true + false + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + + + true + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/symreader.proj b/src/SourceBuild/tarball/content/repos/symreader.proj new file mode 100644 index 000000000..0e8284c7e --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/symreader.proj @@ -0,0 +1,47 @@ + + + + + + $(ProjectDirectory)global.json + $(ProjectDirectory)nuget.config + $(ProjectDirectory)/artifacts/packages/$(Configuration)/Shipping + false + true + + + + + + + + + + + + + + + $(ProjectDirectory)/src//Microsoft.DiaSymReader//Microsoft.DiaSymReader.csproj + $(BuildCommandArgs) /p:Configuration=$(Configuration) + $(BuildCommandArgs) /v:$(LogVerbosity) + $(BuildCommandArgs) $(RedirectRepoOutputToLog) + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/templating.proj b/src/SourceBuild/tarball/content/repos/templating.proj new file mode 100644 index 000000000..5cea61baf --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/templating.proj @@ -0,0 +1,53 @@ + + + + + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + $(ProjectDirectory)artifacts/packages/$(Configuration)/NonShipping/ + false + + --restore --pack --configuration $(Configuration) $(OutputVersionArgs) + $(BuildCommandArgs) $(FlagParameterPrefix)warnasError $(ArcadeFalseBoolBuildArg) + $(BuildCommandArgs) /p:PackSpecific=true + $(BuildCommandArgs) /p:UseAppHost=false + + true + $(BuildCommandArgs) -v $(LogVerbosity) + $(BuildCommandArgs) -bl + $(BuildCommandArgs) --ci + $(BuildCommandArgs) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg) + + + $(BuildCommandArgs) /p:PB_PackageVersionPropsUrl=file:%2F%2F$(PackageVersionPropsPath) + + $(ProjectDirectory)\build$(ShellExtension) $(BuildCommandArgs) + + true + + $(SourceBuiltPackagesPath) + $(EnvironmentRestoreSources)%3B$(ReferencePackagesDir)%3B$(PrebuiltPackagesPath) + $(EnvironmentRestoreSources)%3Bhttps://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public%40Local/nuget/v3/index.json + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/test-templates.proj b/src/SourceBuild/tarball/content/repos/test-templates.proj new file mode 100644 index 000000000..2951834f1 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/test-templates.proj @@ -0,0 +1,29 @@ + + + + + + --restore --build --pack + $(BuildCommandArgs) --configuration $(Configuration) + $(BuildCommandArgs) --binaryLog + $(BuildCommandArgs) -ci + $(BuildCommandArgs) $(FlagParameterPrefix)nodereuse $(ArcadeFalseBoolBuildArg) + $(ProjectDirectory)build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)global.json + $(ProjectDirectory)NuGet.config + true + false + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/vstest.proj b/src/SourceBuild/tarball/content/repos/vstest.proj new file mode 100644 index 000000000..e1feeb741 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/vstest.proj @@ -0,0 +1,31 @@ + + + + + $(TargetRid) + freebsd-x64 + + -DotNetBuildFromSource + $(BuildCommandArgs) -DotNetCoreSdkDir $(DotNetCliToolDir) + $(BuildCommandArgs) -c $(Configuration) + $(BuildCommandArgs) -r $(OverrideTargetRid) + $(BuildCommandArgs) -v $(OutputPackageVersion) + $(BuildCommandArgs) $(FlagParameterPrefix)warnAsError $(ArcadeFalseBoolBuildArg) + + $(ProjectDirectory)/build$(ShellExtension) $(BuildCommandArgs) + + $(ProjectDirectory)/artifacts/$(Configuration)/packages + false + true + $(ProjectDirectory)global.json + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/repos/xliff-tasks.proj b/src/SourceBuild/tarball/content/repos/xliff-tasks.proj new file mode 100644 index 000000000..849340be0 --- /dev/null +++ b/src/SourceBuild/tarball/content/repos/xliff-tasks.proj @@ -0,0 +1,35 @@ + + + + + pack + $(BuildCommandArgs) $(ProjectDirectory)src\XliffTasks\XliffTasks.csproj + $(BuildCommandArgs) /v:$(LogVerbosity) + $(BuildCommandArgs) /flp:Verbosity=Diag + $(BuildCommandArgs) /bl + $(BuildCommandArgs) /p:TreatWarningsAsErrors=false + $(BuildCommandArgs) $(RedirectRepoOutputToLog) + + $(DotnetToolCommand) $(BuildCommandArgs) + + $(ProjectDirectory)artifacts/packages/$(Configuration)/Shipping/ + false + true + true + true + + $(ProjectDirectory)/NuGet.config + $(SourceBuiltPackagesPath) + $(EnvironmentRestoreSources)%3B$(ReferencePackagesDir)%3B$(PrebuiltPackagesPath) + $(EnvironmentRestoreSources)%3Bhttps://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-public%40Local/nuget/v3/index.json + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/scripts/bootstrap/buildbootstrapcli.sh b/src/SourceBuild/tarball/content/scripts/bootstrap/buildbootstrapcli.sh new file mode 100755 index 000000000..b27b5476e --- /dev/null +++ b/src/SourceBuild/tarball/content/scripts/bootstrap/buildbootstrapcli.sh @@ -0,0 +1,411 @@ +#!/usr/bin/env bash +set -e +set -u +set -o pipefail + +usage() +{ + echo "Builds a bootstrap CLI from sources" + echo "Usage: $0 [BuildType] -rid -seedcli [-os ] [-clang ] [-corelib ]" + echo "" + echo "Options:" + echo " BuildType Type of build (-debug, -release), default: -release" + echo " -clang Override of the version of clang compiler to use" + echo " -corelib Path to System.Private.CoreLib.dll, default: use the System.Private.CoreLib.dll from the seed CLI" + echo " -os Operating system (used for corefx build), default: Linux" + echo " -rid Runtime identifier including the architecture part (e.g. rhel.6-x64)" + echo " -seedcli Seed CLI used to generate the target CLI" + echo " -outputpath Optional output directory to contain the generated cli and cloned repos, default: " +} + +disable_pax_mprotect() +{ + if [[ $(command -v paxctl) ]]; then + paxctl -c -m $1 + fi +} + +get_max_version() +{ + local maxversionhi=0 + local maxversionmid=0 + local maxversionlo=0 + local maxversiontag + local versionrest + local versionhi + local versionmid + local versionlo + local versiontag + local foundmax + + for d in $1/*; do + + if [[ -d $d ]]; then + versionrest=$(basename $d) + versionhi=${versionrest%%.*} + versionrest=${versionrest#*.} + versionmid=${versionrest%%.*} + versionrest=${versionrest#*.} + versionlo=${versionrest%%-*} + versiontag=${versionrest#*-} + if [[ $versiontag == $versionrest ]]; then + versiontag="" + fi + + foundmax=0 + + if [[ $versionhi -gt $maxversionhi ]]; then + foundmax=1 + elif [[ $versionhi -eq $maxversionhi ]]; then + if [[ $versionmid -gt $maxversionmid ]]; then + foundmax=1 + elif [[ $versionmid -eq $maxversionmid ]]; then + if [[ $versionlo -gt $maxversionlo ]]; then + foundmax=1 + elif [[ $versionlo -eq $maxversionlo ]]; then + # tags are used to mark pre-release versions, so a version without a tag + # is newer than a version with one. + if [[ "$versiontag" == "" || $versiontag > $maxversiontag ]]; then + foundmax=1 + fi + fi + fi + fi + + if [[ $foundmax != 0 ]]; then + maxversionhi=$versionhi + maxversionmid=$versionmid + maxversionlo=$versionlo + maxversiontag=$versiontag + fi + fi + done + + echo $maxversionhi.$maxversionmid.$maxversionlo${maxversiontag:+-$maxversiontag} +} + +getrealpath() +{ + if command -v realpath > /dev/null; then + realpath $1 + else + readlink -e $1 + fi +} + +__build_os=Linux +__runtime_id= +__corelib= +__configuration=release +__clangversion= +__outputpath= + +while [[ "${1:-}" != "" ]]; do + lowerI="$(echo $1 | awk '{print tolower($0)}')" + case $lowerI in + -h|--help) + usage + exit 1 + ;; + -rid) + shift + __runtime_id=$1 + ;; + -os) + shift + __build_os=$1 + ;; + -debug) + __configuration=debug + ;; + -release) + __configuration=release + ;; + -corelib) + shift + __corelib=`getrealpath $1` + ;; + -seedcli) + shift + __seedclipath=`getrealpath $1` + ;; + -clang) + shift + __clangversion=clang$1 + ;; + -outputpath) + shift + __outputpath=$1 + ;; + *) + echo "Unknown argument to build.sh $1"; exit 1 + esac + shift +done + + +if [[ -z "$__runtime_id" ]]; then + echo "Missing the required -rid argument" + exit 2 +fi + +if [[ -z "$__seedclipath" ]]; then + echo "Missing the required -seedcli argument" + exit 3 +fi + +__build_arch=${__runtime_id#*-} + +if [[ -z "$__outputpath" ]]; then + __outputpath=$__runtime_id/dotnetcli +fi + +if [[ -d "$__outputpath" ]]; then + /bin/rm -r $__outputpath +fi + +mkdir -p $__runtime_id +mkdir -p $__outputpath + +__outputpath=`getrealpath $__outputpath` + +cd $__runtime_id + +cp -r $__seedclipath/* $__outputpath + +__frameworkversion="2.0.0" +__sdkversion="2.0.0" +__fxrversion="2.0.0" + +echo "**** DETECTING VERSIONS IN SEED CLI ****" + +__frameworkversion=`get_max_version $__seedclipath/shared/Microsoft.NETCore.App` +__sdkversion=`get_max_version $__seedclipath/sdk` +__fxrversion=`get_max_version $__seedclipath/host/fxr` + +echo "Framework version: $__frameworkversion" +echo "SDK version: $__sdkversion" +echo "FXR version: $__fxrversion" + +__frameworkpath=$__outputpath/shared/Microsoft.NETCore.App/$__frameworkversion + +echo "**** DETECTING GIT COMMIT HASHES ****" + +# Extract the git commit hashes representig the state of the three repos that +# the seed cli package was built from +__coreclrhash=`strings $__seedclipath/shared/Microsoft.NETCore.App/$__frameworkversion/libcoreclr.so | grep "@(#)" | grep -o "[a-f0-9]\{40\}"` +__corefxhash=`strings $__seedclipath/shared/Microsoft.NETCore.App/$__frameworkversion/System.Native.so | grep "@(#)" | grep -o "[a-f0-9]\{40\}"` +__coresetuphash=`strings $__seedclipath/dotnet | grep "@(#)" | grep -o "[a-f0-9]\{40\}"` +if [[ "$__coresetuphash" == "" ]]; then + __coresetuphash=`strings $__seedclipath/dotnet | grep -o "[a-f0-9]\{40\}"` +fi + +echo "coreclr hash: $__coreclrhash" +echo "corefx hash: $__corefxhash" +echo "core-setup hash: $__coresetuphash" + +# Clone the three repos if they were not cloned yet. If the folders already +# exist, leave them alone. This allows patching the cloned sources as needed + +if [[ ! -d coreclr ]]; then + echo "**** CLONING CORECLR REPOSITORY ****" + git clone https://github.com/dotnet/coreclr.git + cd coreclr + git checkout $__coreclrhash + cd .. +fi + +if [[ ! -d corefx ]]; then + echo "**** CLONING COREFX REPOSITORY ****" + git clone https://github.com/dotnet/corefx.git + cd corefx + git checkout $__corefxhash + cd .. +fi + +if [[ ! -d core-setup ]]; then + echo "**** CLONING CORE-SETUP REPOSITORY ****" + git clone https://github.com/dotnet/core-setup.git + cd core-setup + git checkout $__coresetuphash + cd .. +fi + +echo "**** BUILDING CORE-SETUP NATIVE COMPONENTS ****" +cd core-setup +src/corehost/build.sh --configuration $__configuration --arch "$__build_arch" --hostver "2.0.0" --apphostver "2.0.0" --fxrver "2.0.0" --policyver "2.0.0" --commithash `git rev-parse HEAD` +cd .. + +echo "**** BUILDING CORECLR NATIVE COMPONENTS ****" +cd coreclr +./build.sh $__configuration $__build_arch $__clangversion -skipgenerateversion -skipmanaged -skipmscorlib -skiprestore -skiprestoreoptdata -skipnuget -nopgooptimize 2>&1 | tee coreclr.log +export __coreclrbin=$(cat coreclr.log | sed -n -e 's/^.*Product binaries are available at //p') +cd .. +echo "CoreCLR binaries will be copied from $__coreclrbin" + +echo "**** BUILDING COREFX NATIVE COMPONENTS ****" +corefx/src/Native/build-native.sh $__build_arch $__configuration $__clangversion $__build_os 2>&1 | tee corefx.log +export __corefxbin=$(cat corefx.log | sed -n -e 's/^.*Build files have been written to: //p') +echo "CoreFX binaries will be copied from $__corefxbin" + +echo "**** Copying new binaries to dotnetcli/ ****" + +# First copy the coreclr repo binaries +cp $__coreclrbin/*so $__frameworkpath +cp $__coreclrbin/corerun $__frameworkpath +cp $__coreclrbin/crossgen $__frameworkpath + +# Mark the coreclr executables as allowed to create executable memory mappings +disable_pax_mprotect $__frameworkpath/corerun +disable_pax_mprotect $__frameworkpath/crossgen + +# Now copy the core-setup repo binaries + +if [[ $__fxrversion == 2* ]]; then + cp core-setup/cli/exe/dotnet/dotnet $__outputpath + cp core-setup/cli/exe/dotnet/dotnet $__frameworkpath/corehost + + cp core-setup/cli/dll/libhostpolicy.so $__frameworkpath + cp core-setup/cli/dll/libhostpolicy.so $__outputpath/sdk/$__sdkversion + + cp core-setup/cli/fxr/libhostfxr.so $__frameworkpath + cp core-setup/cli/fxr/libhostfxr.so $__outputpath/host/fxr/$__fxrversion + cp core-setup/cli/fxr/libhostfxr.so $__outputpath/sdk/$__sdkversion +else + cp core-setup/bin/$__runtime_id.$__configuration/corehost/dotnet $__outputpath + cp core-setup/bin/$__runtime_id.$__configuration/corehost/dotnet $__frameworkpath/corehost + + cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostpolicy.so $__frameworkpath + cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostpolicy.so $__outputpath/sdk/$__sdkversion + + cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostfxr.so $__frameworkpath + cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostfxr.so $__outputpath/host/fxr/$__fxrversion + cp core-setup/bin/$__runtime_id.$__configuration/corehost/libhostfxr.so $__outputpath/sdk/$__sdkversion +fi + +# Mark the core-setup executables as allowed to create executable memory mappings +disable_pax_mprotect $__outputpath/dotnet +disable_pax_mprotect $__frameworkpath/corehost + +# Finally copy the corefx repo binaries +cp $__corefxbin/**/System.* $__frameworkpath + +# Copy System.Private.CoreLib.dll override from somewhere if requested +if [[ "$__corelib" != "" ]]; then + cp "$__corelib" $__frameworkpath +fi + +# Add the new RID to Microsoft.NETCore.App.deps.json +# Replace the linux-x64 RID in the target, runtimeTarget and runtimes by the new RID +# and add the new RID to the list of runtimes. +echo "**** Adding new rid to Microsoft.NETCore.App.deps.json ****" + +#TODO: add parameter with the parent RID sequence + +sed \ + -e 's/runtime\.linux-x64/runtime.'$__runtime_id'/g' \ + -e 's/runtimes\/linux-x64/runtimes\/'$__runtime_id'/g' \ + -e 's/Version=v\([0-9].[0-9]\)\/linux-x64/Version=v\1\/'$__runtime_id'/g' \ +$__seedclipath/shared/Microsoft.NETCore.App/$__frameworkversion/Microsoft.NETCore.App.deps.json \ +>$__frameworkpath/Microsoft.NETCore.App.deps.json + +# add the new RID to the list of runtimes iff it does not already exist (sed inplace) +__os_dependencies= +if [[ $__build_os == "Linux" ]]; then + __os_dependencies='"linux", "linux-'$__build_arch'", ' +fi +grep -q "\"$__runtime_id\":" $__frameworkpath/Microsoft.NETCore.App.deps.json || \ +sed -i \ + -e 's/"runtimes": {/&\n "'$__runtime_id'": [\n '"$__os_dependencies"'"unix", "unix-'$__build_arch'", "any", "base"\n ],/g' \ +$__frameworkpath/Microsoft.NETCore.App.deps.json + +__crossgentimeout=120 + +function crossgenone(){ + echo $2/crossgen /MissingDependenciesOK /Platform_Assemblies_Paths $2:$3 /in $1 /out $1.ni >$1.log 2>&1 + timeout $__crossgentimeout $2/crossgen /MissingDependenciesOK /Platform_Assemblies_Paths $2:$3 /in $1 /out $1.ni >>$1.log 2>&1 + exitCode=$? + if [ "$exitCode" == "0" ] + then + rm $1.log + mv $1.ni $1 + elif grep -q -e 'The module was expected to contain an assembly manifest' \ + -e 'An attempt was made to load a program with an incorrect format.' \ + -e 'File is PE32' $1.log + then + rm $1.log + echo "$1" >> crossgenskipped + else + echo "$1" >> crossgenretry + fi +} + +# Run an assembly through ildasm ilasm roundtrip to remove x64 crossgen +function uncrossgenone(){ + echo >> $1.log 2>&1 + echo mv $1 $1.x64 >> $1.log 2>&1 + echo $2/ildasm -raweh -out=$1.il $1.x64 "&& \\" >> $1.log 2>&1 + echo $2/ilasm -output=$1 -QUIET -NOLOGO -DEBUG -OPTIMIZE $1.il >> $1.log 2>&1 + + mv $1 $1.x64 + $2/ildasm -raweh -out=$1.il $1.x64 && \ + $2/ilasm -output=$1 -DLL -QUIET -NOLOGO -DEBUG -OPTIMIZE $1.il + exitCode=$? + if [ "$exitCode" == "0" ] + then + rm $1.x64 + rm $1.il + else + echo "$1" >> uncrossgenfails + fi +} + +# if $__build_arch is not x64 then any dll which was crossgened for x64 must be recrossgened for $__build_arch +if [[ "$__build_arch" != "x64" ]] +then + echo "**** Beginning crossgen for $__build_arch target ****" + export -f crossgenone + export __crossgentimeout + + rm -f crossgenretry crossgendlls crossgenskipped uncrossgenfails + + # Assumes System.Private.CoreLib was already crossgened + find $__outputpath -type f -name \*.dll -or -name \*.exe | grep -v System.Private.CoreLib > crossgendlls + + cat crossgendlls | xargs -P 0 -n 1 -I {} bash -c 'crossgenone "$@"' _ {} "$__frameworkpath" "$__outputpath/sdk/$__sdkversion" + + echo + echo "**** Crossgen skipped for non-managed assembly files:" + echo + + touch crossgenskipped + cat crossgenskipped + + echo + echo "**** Crossgen failed for the following dlls:" + echo + + touch crossgenretry + cat crossgenretry + + echo + echo "**** Beginning uncrossgen for failed dlls ****" + echo + export -f uncrossgenone + + rm -f $__coreclrbin/System.Private.CoreLib.dll + ln -s $__corelib $__coreclrbin/System.Private.CoreLib.dll + + cat crossgenretry | xargs -P 0 -n 1 -I {} bash -c 'uncrossgenone "$@"' _ {} "$__coreclrbin" + + rm -f $__coreclrbin/System.Private.CoreLib.dll + + echo + echo "**** Uncrossgen failed for the following dlls:" + echo + touch uncrossgenfails + cat uncrossgenfails +fi + +echo "**** Bootstrap CLI was successfully built ****" + diff --git a/src/SourceBuild/tarball/content/scripts/docker/docker-run.cmd b/src/SourceBuild/tarball/content/scripts/docker/docker-run.cmd new file mode 100644 index 000000000..e7c0d21bb --- /dev/null +++ b/src/SourceBuild/tarball/content/scripts/docker/docker-run.cmd @@ -0,0 +1,47 @@ + +@echo off +setlocal + +set SCRIPT_ROOT=%~dp0 +set REPO_ROOT=%SCRIPT_ROOT%..\..\ + +:arg_loop +set SET_DOCKERFILE= +set SET_DOCKERIMAGE= +if /I "%1" equ "-d" (set SET_DOCKERFILE=1) +if /I "%1" equ "--dockerfile" (set SET_DOCKERFILE=1) +if "%SET_DOCKERFILE%" == "1" ( + echo "1: %1 2: %2" + set DOCKER_FILE=%2 + shift /1 + shift /1 + goto :arg_loop +) +if /I "%1" equ "-i" (set SET_DOCKERIMAGE=1) +if /I "%1" equ "--image" (set SET_DOCKERIMAGE=1) +if "%SET_DOCKERIMAGE%" == "1" ( + set DOCKER_IMAGE=%2 + shift /1 + shift /1 + goto :arg_loop +) + +if "%DOCKER_FILE%" == "" ( + echo Missing required parameter --dockerfile [docker file dir] + exit /b 1 +) +if "%DOCKER_IMAGE%" == "" ( + echo Missing required parameter --image [image name] + exit /b 1 +) + +if EXIST "%DOCKER_FILE%\Dockerfile" ( + docker build -q -f %DOCKER_FILE%\Dockerfile -t %DOCKER_IMAGE% %DOCKER_FILE% +) else ( + echo Error: %DOCKER_FILE%\Dockerfile does not exist + exit /b 1 +) + +docker run -i -t --rm --init -v %REPO_ROOT%:/code -t -w /code %DOCKER_IMAGE% /bin/sh +endlocal + diff --git a/src/SourceBuild/tarball/content/scripts/docker/docker-run.sh b/src/SourceBuild/tarball/content/scripts/docker/docker-run.sh new file mode 100644 index 000000000..93e0382a0 --- /dev/null +++ b/src/SourceBuild/tarball/content/scripts/docker/docker-run.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +set -euo pipefail +IFS=$'\n\t' + +DOCKER_FILE="" +DOCKER_IMAGE="" +SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)" +REPO_ROOT="$(cd -P "$SCRIPT_ROOT/../../" && pwd)" + +case $(echo $1 | awk '{print tolower($0)}') in + -d | --dockerfile) + DOCKER_FILE=$2 + ;; + -i | --image) + DOCKER_IMAGE=$2 + ;; + *) + echo "usage: $0 [[-d | --dockerfile] ] | [-i | --image] ]] cmd-to-run" + exit 1 + ;; +esac + +shift +shift + +if [ $DOCKER_FILE ]; then + if [ -d $DOCKER_FILE ]; then + DOCKER_FILE="$DOCKER_FILE/Dockerfile" + fi + + DOCKER_FILE_DIR=$(dirname $DOCKER_FILE) + DOCKER_IMAGE=$(set -x ; docker build -q -f $DOCKER_FILE $DOCKER_FILE_DIR) +fi + +DOCKER_USERADD_AND_SWITCH_CMD="" + +if [ ! $(id -u) = 0 ]; then + DOCKER_USERADD_AND_SWITCH_CMD="useradd -m -u $(id -u) $(id -n -u) && su $(id -n -u) -c " +fi + +ARGS=$(IFS=' ' ; echo $@) +(set -x ; docker run --rm --init -v $REPO_ROOT:/code -t $DOCKER_IMAGE /bin/sh -c "cd /code ; $DOCKER_USERADD_AND_SWITCH_CMD\"$ARGS\"") diff --git a/src/SourceBuild/tarball/content/scripts/generate-readme-table.sh b/src/SourceBuild/tarball/content/scripts/generate-readme-table.sh new file mode 100644 index 000000000..244306d7d --- /dev/null +++ b/src/SourceBuild/tarball/content/scripts/generate-readme-table.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash +set -euo pipefail + +script_root="$(cd -P "$( dirname "$0" )" && pwd)" + +branch=master +branch_azdo=$branch + +readme="$script_root/../README.md" + +if [ ! -f "$readme" ]; then + echo "$readme must exist." + exit 1 +fi + +print_rows() { + echo '| OS | *Azure DevOps*
Release |' + echo '| -- | :-- |' + row 'CentOS7.1' 'Production' + row 'CentOS7.1' 'Online' + row 'CentOS7.1' 'Offline' + row 'CentOS7.1' 'Offline Portable' + row 'Debian8.2' 'Production' + row 'Debian8.2' 'Online' + row 'Fedora29' 'Production' + row 'Fedora29' 'Online' + row 'Fedora29' 'Offline' + row 'Fedora29' 'Offline Portable' + row 'OSX' 'Production' + row 'Ubuntu16.04' 'Production' + row 'Windows' 'Production' +} + +raw_print() { + printf '%s' "$1" +} + +row() { + os=$1 + job_type=$2 + display_name=$os + if [ "$job_type" != "Production" ]; then + display_name="$display_name ($job_type)" + fi + printf "| $display_name | " + azdo + end +} + +end() { + printf '\n' +} + +azdo() { + job=$(raw_print $os | awk '{print tolower($0)}' | sed 's/\.//g') + + # Fix case: AzDO has "sticky" casing across build def lifetime, so these names are inconsistent. + # https://dev.azure.com/dnceng/internal/_workitems/edit/98 + case $os in + OSX|Windows) + job=$os + ;; + esac + + job_type_escaped=$(raw_print "$job_type" | sed 's/ /%20/g') + query="?branchName=$branch_azdo&jobname=$job&configuration=$job_type_escaped" + + raw_print "[![Build Status](https://dev.azure.com/dnceng/internal/_apis/build/status/dotnet/source-build/source-build-CI$query)]" + raw_print "(https://dev.azure.com/dnceng/internal/_build/latest?definitionId=114&branchName=$branch_azdo) | " +} + +none() { + raw_print '| ' +} + +cp "$readme" "$readme.old" + +phase=before +while read line; do + if [ "$phase" = before ]; then + echo "$line" + if [ "$line" = '' ]; then + print_rows + phase=skip + fi + elif [ "$phase" = skip ]; then + if [ "$line" = '' ]; then + echo "$line" + phase=after + fi + else + echo "$line" + fi +done < "$readme.old" > "$readme" + +rm "$readme.old" diff --git a/src/SourceBuild/tarball/content/smoke-test.sh b/src/SourceBuild/tarball/content/smoke-test.sh new file mode 100755 index 000000000..c97da1044 --- /dev/null +++ b/src/SourceBuild/tarball/content/smoke-test.sh @@ -0,0 +1,674 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)" +TARBALL_PREFIX=dotnet-sdk- +VERSION_PREFIX=5.0 +# See https://github.com/dotnet/source-build/issues/579, this version +# needs to be compatible with the runtime produced from source-build +DEV_CERTS_VERSION_DEFAULT=5.0.0-preview.3 +__ROOT_REPO=$(sed 's/\r$//' "$SCRIPT_ROOT/artifacts/obj/rootrepo.txt") # remove CR if mounted repo on Windows drive +executingUserHome=${HOME:-} + +export DOTNET_CLI_TELEMETRY_OPTOUT=1 +export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1 + +# Use uname to determine what the CPU is. +cpuName=$(uname -p) +# Some Linux platforms report unknown for platform, but the arch for machine. +if [[ "$cpuName" == "unknown" ]]; then + cpuName=$(uname -m) +fi + +case $cpuName in + aarch64) + buildArch=arm64 + ;; + amd64|x86_64) + buildArch=x64 + ;; + armv*l) + buildArch=arm + ;; + i686) + buildArch=x86 + ;; + *) + echo "Unknown CPU $cpuName detected, treating it as x64" + buildArch=x64 + ;; +esac + +projectOutput=false +keepProjects=false +dotnetDir="" +configuration="Release" +excludeNonWebTests=false +excludeWebTests=false +excludeWebNoHttpsTests=false +excludeWebHttpsTests=false +excludeLocalTests=false +excludeOnlineTests=false +devCertsVersion="$DEV_CERTS_VERSION_DEFAULT" +testingDir="$SCRIPT_ROOT/testing-smoke" +cliDir="$testingDir/builtCli" +logFile="$testingDir/smoke-test.log" +restoredPackagesDir="$testingDir/packages" +testingHome="$testingDir/home" +archiveRestoredPackages=false +archivedPackagesDir="$testingDir/smoke-test-packages" +smokeTestPrebuilts="$SCRIPT_ROOT/packages/smoke-test-packages" +runningOnline=false +runningHttps=false + +function usage() { + echo "" + echo "usage:" + echo " --dotnetDir the directory from which to run dotnet" + echo " --configuration the configuration being tested (default=Release)" + echo " --targetRid override the target rid to use when needed (e.g. for self-contained publish tests)" + echo " --projectOutput echo dotnet's output to console" + echo " --keepProjects keep projects after tests are complete" + echo " --minimal run minimal set of tests - local sources only, no web" + echo " --excludeNonWebTests don't run tests for non-web projects" + echo " --excludeWebTests don't run tests for web projects" + echo " --excludeWebNoHttpsTests don't run web project tests with --no-https" + echo " --excludeWebHttpsTests don't run web project tests with https using dotnet-dev-certs" + echo " --excludeLocalTests exclude tests that use local sources for nuget packages" + echo " --excludeOnlineTests exclude test that use online sources for nuget packages" + echo " --devCertsVersion use dotnet-dev-certs instead of default $DEV_CERTS_VERSION_DEFAULT" + echo " --prodConBlobFeedUrl override the prodcon blob feed specified in ProdConFeed.txt, removing it if empty" + echo " --archiveRestoredPackages capture all restored packages to $archivedPackagesDir" + echo "environment:" + echo " prodConBlobFeedUrl override the prodcon blob feed specified in ProdConFeed.txt, removing it if empty" + echo "" +} + +while :; do + if [ $# -le 0 ]; then + break + fi + + lowerI="$(echo "$1" | awk '{print tolower($0)}')" + case $lowerI in + '-?'|-h|--help) + usage + exit 0 + ;; + --dotnetdir) + shift + dotnetDir="$1" + ;; + --configuration) + shift + configuration="$1" + ;; + --targetrid) + shift + targetRid="$1" + ;; + --projectoutput) + projectOutput=true + ;; + --keepprojects) + keepProjects=true + ;; + --minimal) + excludeOnlineTests=true + ;; + --excludenonwebtests) + excludeNonWebTests=true + ;; + --excludewebtests) + excludeWebTests=true + ;; + --excludewebnohttpstests) + excludeWebNoHttpsTests=true + ;; + --excludewebhttpstests) + excludeWebHttpsTests=true + ;; + --excludelocaltests) + excludeLocalTests=true + ;; + --excludeonlinetests) + excludeOnlineTests=true + ;; + --devcertsversion) + shift + devCertsVersion="$1" + ;; + --prodconblobfeedurl) + shift + prodConBlobFeedUrl="$1" + ;; + --archiverestoredpackages) + archiveRestoredPackages=true + ;; + *) + echo "Unrecognized argument '$1'" + usage + exit 1 + ;; + esac + + shift +done + +prodConBlobFeedUrl="${prodConBlobFeedUrl-$(cat "$SCRIPT_ROOT/ProdConFeed.txt")}" + +function doCommand() { + lang=$1 + proj=$2 + shift; shift; + + echo "starting language $lang, type $proj" | tee -a smoke-test.log + + dotnetCmd=${dotnetDir}/dotnet + mkdir "${lang}_${proj}" + cd "${lang}_${proj}" + + newArgs="new $proj -lang $lang" + + while :; do + if [ $# -le 0 ]; then + break + fi + case "$1" in + --new-arg) + shift + newArgs="$newArgs $1" + ;; + *) + break + ;; + esac + shift + done + + while :; do + if [ $# -le 0 ]; then + break + fi + + binlogOnlinePart="local" + binlogHttpsPart="nohttps" + if [ "$runningOnline" == "true" ]; then + binlogOnlinePart="online" + fi + if [ "$runningHttps" == "true" ]; then + binlogHttpsPart="https" + fi + + binlogPrefix="$testingDir/${lang}_${proj}_${binlogOnlinePart}_${binlogHttpsPart}_" + binlog="${binlogPrefix}$1.binlog" + echo " running $1" | tee -a "$logFile" + + if [ "$1" == "new" ]; then + if [ "$projectOutput" == "true" ]; then + "${dotnetCmd}" $newArgs --no-restore | tee -a "$logFile" + else + "${dotnetCmd}" $newArgs --no-restore >> "$logFile" 2>&1 + fi + elif [[ "$1" == "run" && "$proj" =~ ^(web|mvc|webapi|razor|blazorwasm|blazorserver)$ ]]; then + # A separate log file that we will over-write all the time. + exitLogFile="$testingDir/exitLogFile" + echo > "$exitLogFile" + # Run an application in the background and redirect its + # stdout+stderr to a separate process (tee). The tee process + # writes its input to 2 files: + # - Either the normal log or stdout + # - A log that's only used to find out when it's safe to kill + # the application. + if [ "$projectOutput" == "true" ]; then + "${dotnetCmd}" $1 2>&1 > >(tee -a "$exitLogFile") & + else + "${dotnetCmd}" $1 2>&1 > >(tee -a "$logFile" "$exitLogFile" >/dev/null) & + fi + webPid=$! + killCommand="pkill -SIGTERM -P $webPid" + echo " waiting up to 30 seconds for web project with pid $webPid..." + echo " to clean up manually after an interactive cancellation, run: $killCommand" + for seconds in $(seq 30); do + if grep 'Application started. Press Ctrl+C to shut down.' "$exitLogFile"; then + echo " app ready for shutdown after $seconds seconds" + break + fi + sleep 1 + done + echo " stopping $webPid" | tee -a "$logFile" + $killCommand + wait $! + echo " terminated with exit code $?" | tee -a "$logFile" + elif [ "$1" == "multi-rid-publish" ]; then + runPublishScenarios() { + "${dotnetCmd}" publish --self-contained false /bl:"${binlogPrefix}publish-fx-dep.binlog" + "${dotnetCmd}" publish --self-contained true -r "$targetRid" /bl:"${binlogPrefix}publish-self-contained-${targetRid}.binlog" + "${dotnetCmd}" publish --self-contained true -r linux-x64 /bl:"${binlogPrefix}publish-self-contained-portable.binlog" + } + if [ "$projectOutput" == "true" ]; then + runPublishScenarios | tee -a "$logFile" + else + runPublishScenarios >> "$logFile" 2>&1 + fi + else + if [ "$projectOutput" == "true" ]; then + "${dotnetCmd}" $1 /bl:"$binlog" | tee -a "$logFile" + else + "${dotnetCmd}" $1 /bl:"$binlog" >> "$logFile" 2>&1 + fi + fi + if [ $? -eq 0 ]; then + echo " $1 succeeded" >> "$logFile" + else + echo " $1 failed with exit code $?" | tee -a "$logFile" + fi + + shift + done + + cd .. + + if [ "$keepProjects" == "false" ]; then + rm -rf "${lang}_${proj}" + fi + + echo "finished language $lang, type $proj" | tee -a smoke-test.log +} + +function setupDevCerts() { + echo "Setting up dotnet-dev-certs $devCertsVersion to generate dev certificate" | tee -a "$logFile" + ( + set -x + "$dotnetDir/dotnet" tool install -g dotnet-dev-certs --version "$devCertsVersion" --add-source https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json + export DOTNET_ROOT="$dotnetDir" + "$testingHome/.dotnet/tools/dotnet-dev-certs" https + ) >> "$logFile" 2>&1 +} + +function runAllTests() { + # Run tests for each language and template + if [ "$excludeNonWebTests" == "false" ]; then + doCommand C# console new restore build run multi-rid-publish + doCommand C# classlib new restore build multi-rid-publish + doCommand C# xunit new restore test + doCommand C# mstest new restore test + + doCommand VB console new restore build run multi-rid-publish + doCommand VB classlib new restore build multi-rid-publish + doCommand VB xunit new restore test + doCommand VB mstest new restore test + + doCommand F# console new restore build run multi-rid-publish + doCommand F# classlib new restore build multi-rid-publish + doCommand F# xunit new restore test + doCommand F# mstest new restore test + fi + + if [ "$excludeWebTests" == "false" ]; then + if [ "$excludeWebNoHttpsTests" == "false" ]; then + runningHttps=false + runWebTests --new-arg --no-https + fi + + if [ "$excludeWebHttpsTests" == "false" ]; then + runningHttps=true + setupDevCerts + runWebTests + fi + fi +} + +function runWebTests() { + doCommand C# web "$@" new restore build run multi-rid-publish + doCommand C# mvc "$@" new restore build run multi-rid-publish + doCommand C# webapi "$@" new restore build multi-rid-publish + doCommand C# razor "$@" new restore build run multi-rid-publish + doCommand C# blazorwasm "$@" new restore build run publish + doCommand C# blazorserver "$@" new restore build run publish + + doCommand F# web "$@" new restore build run multi-rid-publish + doCommand F# mvc "$@" new restore build run multi-rid-publish + doCommand F# webapi "$@" new restore build run multi-rid-publish +} + +function runXmlDocTests() { + targetingPacksDir="$dotnetDir/packs/" + echo "Looking for xml docs in targeting packs in $targetingPacksDir" + + netstandardIgnoreList=( + Microsoft.Win32.Primitives.xml + mscorlib.xml + System.AppContext.xml + System.Buffers.xml + System.Collections.Concurrent.xml + System.Collections.NonGeneric.xml + System.Collections.Specialized.xml + System.Collections.xml + System.ComponentModel.Composition.xml + System.ComponentModel.EventBasedAsync.xml + System.ComponentModel.Primitives.xml + System.ComponentModel.TypeConverter.xml + System.ComponentModel.xml + System.Console.xml + System.Core.xml + System.Data.Common.xml + System.Data.xml + System.Diagnostics.Contracts.xml + System.Diagnostics.Debug.xml + System.Diagnostics.FileVersionInfo.xml + System.Diagnostics.Process.xml + System.Diagnostics.StackTrace.xml + System.Diagnostics.TextWriterTraceListener.xml + System.Diagnostics.Tools.xml + System.Diagnostics.TraceSource.xml + System.Diagnostics.Tracing.xml + System.Drawing.Primitives.xml + System.Drawing.xml + System.Dynamic.Runtime.xml + System.Globalization.Calendars.xml + System.Globalization.Extensions.xml + System.Globalization.xml + System.IO.Compression.FileSystem.xml + System.IO.Compression.xml + System.IO.Compression.ZipFile.xml + System.IO.FileSystem.DriveInfo.xml + System.IO.FileSystem.Primitives.xml + System.IO.FileSystem.Watcher.xml + System.IO.FileSystem.xml + System.IO.IsolatedStorage.xml + System.IO.MemoryMappedFiles.xml + System.IO.Pipes.xml + System.IO.UnmanagedMemoryStream.xml + System.IO.xml + System.Linq.Expressions.xml + System.Linq.Parallel.xml + System.Linq.Queryable.xml + System.Linq.xml + System.Memory.xml + System.Net.Http.xml + System.Net.NameResolution.xml + System.Net.NetworkInformation.xml + System.Net.Ping.xml + System.Net.Primitives.xml + System.Net.Requests.xml + System.Net.Security.xml + System.Net.Sockets.xml + System.Net.WebHeaderCollection.xml + System.Net.WebSockets.Client.xml + System.Net.WebSockets.xml + System.Net.xml + System.Numerics.Vectors.xml + System.Numerics.xml + System.ObjectModel.xml + System.Reflection.DispatchProxy.xml + System.Reflection.Emit.ILGeneration.xml + System.Reflection.Emit.Lightweight.xml + System.Reflection.Emit.xml + System.Reflection.Extensions.xml + System.Reflection.Primitives.xml + System.Reflection.xml + System.Resources.Reader.xml + System.Resources.ResourceManager.xml + System.Resources.Writer.xml + System.Runtime.CompilerServices.VisualC.xml + System.Runtime.Extensions.xml + System.Runtime.Handles.xml + System.Runtime.InteropServices.RuntimeInformation.xml + System.Runtime.InteropServices.xml + System.Runtime.Numerics.xml + System.Runtime.Serialization.Formatters.xml + System.Runtime.Serialization.Json.xml + System.Runtime.Serialization.Primitives.xml + System.Runtime.Serialization.xml + System.Runtime.Serialization.Xml.xml + System.Runtime.xml + System.Security.Claims.xml + System.Security.Cryptography.Algorithms.xml + System.Security.Cryptography.Csp.xml + System.Security.Cryptography.Encoding.xml + System.Security.Cryptography.Primitives.xml + System.Security.Cryptography.X509Certificates.xml + System.Security.Principal.xml + System.Security.SecureString.xml + System.ServiceModel.Web.xml + System.Text.Encoding.Extensions.xml + System.Text.Encoding.xml + System.Text.RegularExpressions.xml + System.Threading.Overlapped.xml + System.Threading.Tasks.Extensions.xml + System.Threading.Tasks.Parallel.xml + System.Threading.Tasks.xml + System.Threading.ThreadPool.xml + System.Threading.Thread.xml + System.Threading.Timer.xml + System.Threading.xml + System.Transactions.xml + System.ValueTuple.xml + System.Web.xml + System.Windows.xml + System.xml + System.Xml.Linq.xml + System.Xml.ReaderWriter.xml + System.Xml.Serialization.xml + System.Xml.XDocument.xml + System.Xml.xml + System.Xml.XmlDocument.xml + System.Xml.XmlSerializer.xml + System.Xml.XPath.XDocument.xml + System.Xml.XPath.xml + ) + + netcoreappIgnoreList=( + Microsoft.VisualBasic.xml + netstandard.xml + System.AppContext.xml + System.Buffers.xml + System.ComponentModel.DataAnnotations.xml + System.Configuration.xml + System.Core.xml + System.Data.DataSetExtensions.xml + System.Data.xml + System.Diagnostics.Debug.xml + System.Diagnostics.Tools.xml + System.Drawing.xml + System.Dynamic.Runtime.xml + System.Globalization.Calendars.xml + System.Globalization.Extensions.xml + System.Globalization.xml + System.IO.Compression.FileSystem.xml + System.IO.FileSystem.Primitives.xml + System.IO.UnmanagedMemoryStream.xml + System.IO.xml + System.Net.xml + System.Numerics.xml + System.Reflection.Extensions.xml + System.Reflection.xml + System.Resources.Reader.xml + System.Resources.ResourceManager.xml + System.Runtime.Extensions.xml + System.Runtime.Handles.xml + System.Runtime.Serialization.xml + System.Security.Principal.xml + System.Security.SecureString.xml + System.Security.xml + System.ServiceModel.Web.xml + System.ServiceProcess.xml + System.Text.Encoding.xml + System.Threading.Tasks.Extensions.xml + System.Threading.Tasks.xml + System.Threading.Timer.xml + System.Transactions.xml + System.ValueTuple.xml + System.Web.xml + System.Windows.xml + System.xml + System.Xml.Linq.xml + System.Xml.Serialization.xml + System.Xml.xml + System.Xml.XmlDocument.xml + ) + + error=0 + while IFS= read -r -d '' dllFile; do + xmlDocFile=${dllFile%.*}.xml + skip=0 + if [[ "$xmlDocFile" == *"/packs/Microsoft.NETCore.App.Ref"* ]]; then + xmlFileBasename=$(basename "$xmlDocFile") + for ignoreItem in "${netcoreappIgnoreList[@]}"; do + if [[ "$ignoreItem" == "$xmlFileBasename" ]]; then + skip=1; + break + fi + done + fi + if [[ "$xmlDocFile" == *"/packs/NETStandard.Library.Ref"* ]]; then + xmlFileBasename=$(basename "$xmlDocFile") + for ignoreItem in "${netstandardIgnoreList[@]}"; do + if [[ "$ignoreItem" == "$xmlFileBasename" ]]; then + skip=1; + break + fi + done + fi + if [[ $skip == 0 ]] && [[ ! -f "$xmlDocFile" ]]; then + error=1 + echo "error: missing $xmlDocFile" + fi + done < <(find "$targetingPacksDir" -name '*.dll' -print0) + + if [[ $error != 0 ]]; then + echo "error: Missing xml documents" + exit 1 + else + echo "All expected xml docs are present" + fi +} + +function resetCaches() { + rm -rf "$testingHome" + mkdir "$testingHome" + + HOME="$testingHome" + + # clean restore path + rm -rf "$restoredPackagesDir" + + # Copy NuGet plugins if running user has HOME and we have auth. In particular, the auth plugin. + if [ "${internalPackageFeedPat:-}" ] && [ "${executingUserHome:-}" ]; then + cp -r "$executingUserHome/.nuget/" "$HOME/.nuget/" || : + fi +} + +function setupProdConFeed() { + if [ "$prodConBlobFeedUrl" ]; then + sed -i.bakProdCon "s|PRODUCT_CONTRUCTION_PACKAGES|$prodConBlobFeedUrl|g" "$testingDir/NuGet.Config" + else + sed -i.bakProdCon "/PRODUCT_CONTRUCTION_PACKAGES/d" "$testingDir/NuGet.Config" + fi +} + +function setupSmokeTestFeed() { + # Setup smoke-test-packages if they exist + if [ -e "$smokeTestPrebuilts" ]; then + sed -i.bakSmokeTestFeed "s|SMOKE_TEST_PACKAGE_FEED|$smokeTestPrebuilts|g" "$testingDir/NuGet.Config" + else + sed -i.bakSmokeTestFeed "/SMOKE_TEST_PACKAGE_FEED/d" "$testingDir/NuGet.Config" + fi +} + +function copyRestoredPackages() { + if [ "$archiveRestoredPackages" == "true" ]; then + mkdir -p "$archivedPackagesDir" + cp -rf "$restoredPackagesDir"/* "$archivedPackagesDir" + fi +} + +echo "RID to test: ${targetRid?not specified. Use ./build.sh --run-smoke-test to detect RID, or specify manually.}" + +if [ "$__ROOT_REPO" != "known-good" ]; then + echo "Skipping smoke-tests since cli was not built"; + exit +fi + +# Clean up and create directory +if [ -e "$testingDir" ]; then + read -p "testing-smoke directory exists, remove it? [Y]es / [n]o" -n 1 -r + echo + if [[ $REPLY == "" || $REPLY == " " || $REPLY =~ ^[Yy]$ ]]; then + rm -rf "$testingDir" + fi +fi + +mkdir -p "$testingDir" +cd "$testingDir" + +# Create blank Directory.Build files to avoid traversing to source-build infra. +echo "" | tee Directory.Build.props > Directory.Build.targets + +# Unzip dotnet if the dotnetDir is not specified +if [ "$dotnetDir" == "" ]; then + OUTPUT_DIR="$SCRIPT_ROOT/artifacts/$buildArch/$configuration/" + DOTNET_TARBALL="$(ls "${OUTPUT_DIR}${TARBALL_PREFIX}${VERSION_PREFIX}"*)" + + mkdir -p "$cliDir" + tar xzf "$DOTNET_TARBALL" -C "$cliDir" + dotnetDir="$cliDir" +else + if ! [[ "$dotnetDir" = /* ]]; then + dotnetDir="$SCRIPT_ROOT/$dotnetDir" + fi +fi + +echo SDK under test is: +"$dotnetDir/dotnet" --info + +# setup restore path +export NUGET_PACKAGES="$restoredPackagesDir" +SOURCE_BUILT_PKGS_PATH="$SCRIPT_ROOT/artifacts/obj/$buildArch/$configuration/blob-feed/packages/" +export DOTNET_ROOT="$dotnetDir" +# OSX also requires DOTNET_ROOT to be on the PATH +if [ "$(uname)" == 'Darwin' ]; then + export PATH="$dotnetDir:$PATH" +fi + +# Run all tests, local restore sources first, online restore sources second +if [ "$excludeLocalTests" == "false" ]; then + resetCaches + runningOnline=false + # Setup NuGet.Config with local restore source + if [ -e "$SCRIPT_ROOT/smoke-testNuGet.Config" ]; then + cp "$SCRIPT_ROOT/smoke-testNuGet.Config" "$testingDir/NuGet.Config" + sed -i.bak "s|SOURCE_BUILT_PACKAGES|$SOURCE_BUILT_PKGS_PATH|g" "$testingDir/NuGet.Config" + setupProdConFeed + setupSmokeTestFeed + echo "$testingDir/NuGet.Config Contents:" + cat "$testingDir/NuGet.Config" + fi + echo "RUN ALL TESTS - LOCAL RESTORE SOURCE" + runAllTests + copyRestoredPackages + echo "LOCAL RESTORE SOURCE - ALL TESTS PASSED!" +fi + +if [ "$excludeOnlineTests" == "false" ]; then + resetCaches + runningOnline=true + # Setup NuGet.Config to use online restore sources + if [ -e "$SCRIPT_ROOT/smoke-testNuGet.Config" ]; then + cp "$SCRIPT_ROOT/smoke-testNuGet.Config" "$testingDir/NuGet.Config" + sed -i.bak "/SOURCE_BUILT_PACKAGES/d" "$testingDir/NuGet.Config" + setupProdConFeed + setupSmokeTestFeed + echo "$testingDir/NuGet.Config Contents:" + cat "$testingDir/NuGet.Config" + fi + echo "RUN ALL TESTS - ONLINE RESTORE SOURCE" + runAllTests + copyRestoredPackages + echo "ONLINE RESTORE SOURCE - ALL TESTS PASSED!" +fi + +runXmlDocTests + +echo "ALL TESTS PASSED!" diff --git a/src/SourceBuild/tarball/content/smoke-testNuGet.Config b/src/SourceBuild/tarball/content/smoke-testNuGet.Config new file mode 100644 index 000000000..07754412d --- /dev/null +++ b/src/SourceBuild/tarball/content/smoke-testNuGet.Config @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/Directory.Build.props b/src/SourceBuild/tarball/content/tools-local/Directory.Build.props new file mode 100644 index 000000000..bffd69d62 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/Directory.Build.props @@ -0,0 +1,13 @@ + + + + + true + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/acquire-darc/acquire-darc.proj b/src/SourceBuild/tarball/content/tools-local/acquire-darc/acquire-darc.proj new file mode 100644 index 000000000..054a65119 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/acquire-darc/acquire-darc.proj @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/generate-graphviz/generate-graphviz.proj b/src/SourceBuild/tarball/content/tools-local/generate-graphviz/generate-graphviz.proj new file mode 100644 index 000000000..888cbd9c6 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/generate-graphviz/generate-graphviz.proj @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + $(BaseIntermediatePath)graphviz.dot + $(BaseIntermediatePath)graphviz.png + digraph { +graph [ dpi = 150 ] +@(RepoLink -> '%(Text)') +} + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/init-build.proj b/src/SourceBuild/tarball/content/tools-local/init-build.proj new file mode 100644 index 000000000..5213830c6 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/init-build.proj @@ -0,0 +1,170 @@ + + + + + + + + + + + + + + + + + + + + $(ReferencePackagesDir)%3B$(PrebuiltPackagesPath) + + + + + + + + + + + + + + + + + + + + + + + + + $(IntermediatePath)PackageVersions.props + + + + + + +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/prebuilt-baseline-offline.xml b/src/SourceBuild/tarball/content/tools-local/prebuilt-baseline-offline.xml new file mode 100644 index 000000000..cca3774b3 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/prebuilt-baseline-offline.xml @@ -0,0 +1,46 @@ + + centos.7-x64 + + src/ApplicationInsights-dotnet.53b80940842204f78708a538628288ff5d741a1d/ + src/arcade.6eec4404c2df5bfa46e5da52383c881c5cca3a9f/ + src/xdt.c01a538851a8ab1a1fbeb2e6243f391fff7587b4/ + src/dotnet-aspnetcore.2670c128d522473e146ff9f8159bfffdfe694cd9/ + src/cliCommandLineParser.0e89c2116ad28e404ba56c14d1c3f938caa25a01/ + src/command-line-api.afd010ba8cb3cbd714c734465d1a5de1ee133e2d/ + src/common.6e37cdfe96ac8b06a923242120169fafacd720e6/ + src/cssparser.d6d86bcd8c162b1ae22ef00955ff748d028dd0ee/ + src/diagnostics.47296ca69bb66180c132f3b16667f904dfc7c6c7/ + src/fsharp.da6be68280c89131cdba2045525b80890401defd/ + src/Humanizer.b30550eed103a6970d8465fe7c5c16300b70be81/ + src/dotnet-installer.71365b4d424b0860ceb9ad8ee47f7cd08c55c362/ + src/known-good-tests./ + src/known-good./ + src/linker.25604250cf2663aed6630e305cc0538aeebda80a/ + src/msbuild.39993bd9d02917993b6147a9973dc3aa6e9b00c7/ + src/netcorecli-fsc/ + src/Newtonsoft.Json.cac0690ad133c5e166ce5642dc71175791404fad/ + src/Newtonsoft.Json.e43dae94c26f0c30e9095327a3a9eac87193923d/ + src/NuGet.Client.830c8be45dbbccd411ecf6080abff0c2c98079cf/ + src/package-source-build/ + src/roslyn-analyzers.77b259353aa44ec1510951a75c6e7ed6e662a001/ + src/roslyn.9ed4b774d20940880de8df1ca8b07508aa01c8cd/ + src/dotnet-runtime.cb5f173b9696d9d00a544b953d95190ab3b56df2/ + src/dotnet-runtime.cb5f173b9696d9d00a544b953d95190ab3b56df2/ + src/dotnet-sdk.ef14c79a16171496e2d972edd9a7874d596f624d/ + src/sourcelink.f175b06862f889474b689a57527e489101c774cc/ + src/symreader.f8a3ba85aed339fb8d08ca26f3876b28c32d58ee/ + src/templating.568c09b99075179607c572f7b9f3ad3d706f1fc3/ + src/test-templates.956e14dedd3a3ac981b320d66c6d389387a2954a/ + src/vstest.212656d7b384a506aa714999f678b3ace82d114e/ + src/xliff-tasks.a52f3d7fb58470749ee4035fbbcb7e63c78b0459/ + Tools/ + tools-local/tasks/ + artifacts/obj/ + + + + + + + + \ No newline at end of file diff --git a/src/SourceBuild/tarball/content/tools-local/prebuilt-baseline-online.xml b/src/SourceBuild/tarball/content/tools-local/prebuilt-baseline-online.xml new file mode 100755 index 000000000..03370c501 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/prebuilt-baseline-online.xml @@ -0,0 +1,457 @@ + + centos.7-x64 + + artifacts/src/ApplicationInsights-dotnet.53b80940842204f78708a538628288ff5d741a1d/ + artifacts/src/arcade.6eec4404c2df5bfa46e5da52383c881c5cca3a9f/ + artifacts/src/xdt.c01a538851a8ab1a1fbeb2e6243f391fff7587b4/ + artifacts/src/dotnet-aspnetcore.b7a2ec8c7ed6b48857af0a69688a73e8c14fe6cb/ + artifacts/src/CliCommandLineParser.0e89c2116ad28e404ba56c14d1c3f938caa25a01/ + artifacts/src/command-line-api.6ddde11ed45e3f4b9d80c97670f347dbfda15c3f/ + artifacts/src/common.6e37cdfe96ac8b06a923242120169fafacd720e6/ + artifacts/src/cssparser.d6d86bcd8c162b1ae22ef00955ff748d028dd0ee/ + artifacts/src/diagnostics.47296ca69bb66180c132f3b16667f904dfc7c6c7/ + artifacts/src/fsharp.7ce7132f1459095e635194d09d6f73265352029a/ + artifacts/src/Humanizer.b30550eed103a6970d8465fe7c5c16300b70be81/ + artifacts/src/dotnet-installer.db7cc87d512335808e3806067f2bf9b31379e1db/ + artifacts/src/known-good-tests./ + artifacts/src/known-good./ + artifacts/src/linker.c43f981eec336c1dc4fd0ead84b5e09db9377d9e/ + artifacts/src/msbuild.5e4b48a27efce55a613664b58d353ab4c8d1f6c1/ + src/netcorecli-fsc/ + artifacts/src/Newtonsoft.Json.cac0690ad133c5e166ce5642dc71175791404fad/ + artifacts/src/Newtonsoft.Json.e43dae94c26f0c30e9095327a3a9eac87193923d/ + artifacts/src/nuget.client.d525b0e670f3b6cbd5c73a35f04730a9f658c852/ + src/package-source-build/ + artifacts/src/roslyn-analyzers.ce71b27be743710012c0460071da188b2f05959c/ + artifacts/src/roslyn.59eedc33d35754759994155ea2f4e1012a9951e3/ + artifacts/src/dotnet-runtime.2f740adc1457e8a28c1c072993b66f515977eb51/ + artifacts/src/dotnet-runtime.2f740adc1457e8a28c1c072993b66f515977eb51/ + artifacts/src/dotnet-sdk.51369266643769f9f0c1184e89715cd1045126d0/ + artifacts/src/sourcelink.f175b06862f889474b689a57527e489101c774cc/ + artifacts/src/symreader.f8a3ba85aed339fb8d08ca26f3876b28c32d58ee/ + artifacts/src/templating.8470ff317250d761c72f920b8ea1c0700b230eb3/ + artifacts/src/test-templates.956e14dedd3a3ac981b320d66c6d389387a2954a/ + artifacts/src/vstest.99b911a57a02fc5d2eeef23e9ab8cbea4505678b/ + artifacts/src/xliff-tasks.a52f3d7fb58470749ee4035fbbcb7e63c78b0459/ + Tools/ + tools-local/tasks/ + artifacts/obj/ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Directory.Build.props b/src/SourceBuild/tarball/content/tools-local/tasks/Directory.Build.props new file mode 100644 index 000000000..65f8365de --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Directory.Build.props @@ -0,0 +1,29 @@ + + + + + + AnyCPU + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogFileEntry.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogFileEntry.cs new file mode 100644 index 000000000..268d50fc5 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogFileEntry.cs @@ -0,0 +1,26 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + internal class CatalogFileEntry + { + const string ElementName = "File"; + + internal string Path { get; set; } + internal byte[] OriginalHash { get; set; } + internal byte[] PoisonedHash { get; set; } + + public XElement ToXml() => new XElement(ElementName, + new XAttribute(nameof(Path), Path), + new XAttribute(nameof(OriginalHash), OriginalHash.ToHexString()), + PoisonedHash == null ? null : new XAttribute(nameof(PoisonedHash), PoisonedHash.ToHexString()) + ); + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogPackageEntry.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogPackageEntry.cs new file mode 100644 index 000000000..e52e1aaa2 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CatalogPackageEntry.cs @@ -0,0 +1,38 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + internal class CatalogPackageEntry + { + const string ElementName = "Package"; + + internal string Path { get; set; } + internal string Id { get; set; } + internal string Version { get; set; } + internal byte[] OriginalHash { get; set; } + internal byte[] PoisonedHash { get; set; } + internal List Files { get; } + + public CatalogPackageEntry() + { + this.Files = new List(); + } + + public XElement ToXml() => new XElement(ElementName, + new XAttribute(nameof(Path), Path), + new XAttribute(nameof(Id), Id), + new XAttribute(nameof(Version), Version), + new XAttribute(nameof(OriginalHash), OriginalHash.ToHexString()), + PoisonedHash == null ? null : new XAttribute(nameof(PoisonedHash), PoisonedHash.ToHexString()), + Files.Select(f => f.ToXml()) + ); + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs new file mode 100644 index 000000000..3fad37c47 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/CheckForPoison.cs @@ -0,0 +1,384 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Mono.Cecil; +using System; +using System.Collections; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Security.Cryptography; +using System.Text; +using System.Xml; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + public class CheckForPoison : Task + { + /// + /// The files to check for poison and/or hash matches. Zips and + /// nupkgs will be extracted and checked recursively. + /// %(Identity): Path to the initial set of files. + /// + [Required] + public ITaskItem[] FilesToCheck { get; set; } + + /// + /// The output path for an XML poison report, if desired. + /// + public string PoisonReportOutputFilePath { get; set; } + + /// + /// The path of a previously-generated file hash catalog, if + /// hash checked is desired. If not, only assembly attributes + /// and package marker file checked will be done. + /// + public string HashCatalogFilePath { get; set; } + + /// + /// The marker file name to check for in poisoned nupkg files. + /// + public string MarkerFileName { get; set; } + + /// + /// If true, fails the build if any poisoned files are found. + /// + public bool FailOnPoisonFound { get; set; } + + /// + /// Use this directory instead of the system temp directory for staging. + /// Intended for Linux systems with limited /tmp space, like Azure VMs. + /// + public string OverrideTempPath { get; set; } + + private static readonly string[] ZipFileExtensions = + { + ".zip", + ".nupkg", + }; + + private static readonly string[] TarFileExtensions = + { + ".tar", + }; + + private static readonly string[] TarGzFileExtensions = + { + ".tgz", + ".tar.gz", + }; + + private static readonly string[] FileNamesToSkip = + { + "_._", + "-.-", + ".bowerrc", + ".editorconfig", + ".gitignore", + ".gitkeep", + ".rels", + "LICENSE", + "prefercliruntime", + "RunCsc", + "RunVbc", + }; + + private static readonly string[] FileExtensionsToSkip = + { + ".config", + ".cs", + ".cshtml", + ".csproj", + ".css", + ".db", + ".eot", + ".fs", + ".fsproj", + ".html", + ".ico", + ".js", + ".json", + ".map", + ".md", + ".nuspec", + ".png", + ".props", + ".psmdcp", + ".rtf", + ".scss", + ".svg", + ".targets", + ".ts", + ".ttf", + ".txt", + ".vb", + ".vbproj", + ".woff", + ".woff2", + ".xaml", + ".xml", + }; + + private const string PoisonMarker = "POISONED"; + + public override bool Execute() + { + IEnumerable poisons = GetPoisonedFiles(FilesToCheck.Select(f => f.ItemSpec), HashCatalogFilePath, MarkerFileName); + + // if we should write out the poison report, do that + if (!string.IsNullOrWhiteSpace(PoisonReportOutputFilePath)) + { + File.WriteAllText(PoisonReportOutputFilePath, (new XElement("PrebuiltLeakReport", poisons.Select(p => p.ToXml()))).ToString()); + } + + if (FailOnPoisonFound && poisons.Count() > 0) + { + Log.LogError($"Forced build error: {poisons.Count()} marked files leaked to output. See complete report '{PoisonReportOutputFilePath}' for details."); + return false; + } + else if (poisons.Count() > 0) + { + Log.LogWarning($"{poisons.Count()} marked files leaked to output. See complete report '{PoisonReportOutputFilePath}' for details."); + } + else + { + Log.LogError($"No leaked files found in output. Either something is broken or it is the future and we have fixed all leaks - please verify and remove this error if so (and default {nameof(FailOnPoisonFound)} to true)."); + return false; + } + + return !Log.HasLoggedErrors; + } + + /// + /// Internal helper to allow other tasks to check for poisoned files. + /// + /// Initial queue of candidate files (will be cleared when done) + /// File path to the file hash catalog + /// Marker file name to check for in poisoned nupkgs + /// List of poisoned packages and files found and reasons for each + internal IEnumerable GetPoisonedFiles(IEnumerable initialCandidates, string catalogedPackagesFilePath, string markerFileName) + { + IEnumerable catalogedPackages = ReadCatalog(catalogedPackagesFilePath); + var poisons = new List(); + var candidateQueue = new Queue(initialCandidates); + // avoid collisions between nupkgs with the same name + var dirCounter = 0; + if (!string.IsNullOrWhiteSpace(OverrideTempPath)) + { + Directory.CreateDirectory(OverrideTempPath); + } + var tempDirName = Path.GetRandomFileName(); + var tempDir = Directory.CreateDirectory(Path.Combine(OverrideTempPath ?? Path.GetTempPath(), tempDirName)); + + while (candidateQueue.Any()) + { + var checking = candidateQueue.Dequeue(); + + // if this is a zip or NuPkg, extract it, check for the poison marker, and + // add its contents to the list to be checked. + if (ZipFileExtensions.Concat(TarFileExtensions).Concat(TarGzFileExtensions).Any(e => checking.ToLowerInvariant().EndsWith(e))) + { + var tempCheckingDir = Path.Combine(tempDir.FullName, Path.GetRandomFileName(), Path.GetFileNameWithoutExtension(checking) + "." + (++dirCounter).ToString()); + PoisonedFileEntry result = ExtractAndCheckZipFileOnly(catalogedPackages, checking, markerFileName, tempCheckingDir, candidateQueue); + if (result != null) + { + poisons.Add(result); + } + } + else + { + PoisonedFileEntry result = CheckSingleFile(catalogedPackages, tempDir.FullName, checking); + if (result != null) + { + poisons.Add(result); + } + } + } + + tempDir.Delete(true); + + return poisons; + } + + private static PoisonedFileEntry CheckSingleFile(IEnumerable catalogedPackages, string rootPath, string fileToCheck) + { + // skip some common files that get copied verbatim from nupkgs - LICENSE, _._, etc as well as + // file types that we never care about - text files, .gitconfig, etc. + if (FileNamesToSkip.Any(f => Path.GetFileName(fileToCheck).ToLowerInvariant() == f.ToLowerInvariant()) || + FileExtensionsToSkip.Any(e => Path.GetExtension(fileToCheck).ToLowerInvariant() == e.ToLowerInvariant())) + { + return null; + } + + var poisonEntry = new PoisonedFileEntry(); + poisonEntry.Path = Utility.MakeRelativePath(fileToCheck, rootPath); + + // There seems to be some weird issues with using file streams both for hashing and assembly loading. + // Copy everything into a memory stream to avoid these problems. + var memStream = new MemoryStream(); + using (var stream = File.OpenRead(fileToCheck)) + { + stream.CopyTo(memStream); + } + + memStream.Seek(0, SeekOrigin.Begin); + using (var sha = SHA256.Create()) + { + poisonEntry.Hash = sha.ComputeHash(memStream); + } + + foreach (var p in catalogedPackages) + { + // This hash can match either the original hash (we couldn't poison the file, or redownloaded it) or + // the poisoned hash (the obvious failure case of a poisoned file leaked). + foreach (var matchingCatalogedFile in p.Files.Where(f => f.OriginalHash.SequenceEqual(poisonEntry.Hash) || (f.PoisonedHash?.SequenceEqual(poisonEntry.Hash) ?? false))) + { + poisonEntry.Type |= PoisonType.Hash; + var match = new PoisonMatch + { + File = matchingCatalogedFile.Path, + Package = p.Path, + PackageId = p.Id, + PackageVersion = p.Version, + }; + poisonEntry.Matches.Add(match); + } + } + + try + { + memStream.Seek(0, SeekOrigin.Begin); + using (var asm = AssemblyDefinition.ReadAssembly(memStream)) + { + foreach (var a in asm.CustomAttributes) + { + foreach (var ca in a.ConstructorArguments) + { + if (ca.Type.Name == asm.MainModule.TypeSystem.String.Name) + { + if (ca.Value.ToString().Contains(PoisonMarker)) + { + poisonEntry.Type |= PoisonType.AssemblyAttribute; + } + } + } + } + } + } + catch + { + // this is fine, it's just not an assembly. + } + + return poisonEntry.Type != PoisonType.None ? poisonEntry : null; + } + + private static PoisonedFileEntry ExtractAndCheckZipFileOnly(IEnumerable catalogedPackages, string zipToCheck, string markerFileName, string tempDir, Queue futureFilesToCheck) + { + var poisonEntry = new PoisonedFileEntry(); + poisonEntry.Path = zipToCheck; + + using (var sha = SHA256.Create()) + using (var stream = File.OpenRead(zipToCheck)) + { + poisonEntry.Hash = sha.ComputeHash(stream); + } + + // first check for a matching poisoned or non-poisoned hash match: + // - non-poisoned is a potential error where the package was redownloaded. + // - poisoned is a use of a local package we were not expecting. + foreach (var matchingCatalogedPackage in catalogedPackages.Where(c => c.OriginalHash.SequenceEqual(poisonEntry.Hash) || (c.PoisonedHash?.SequenceEqual(poisonEntry.Hash) ?? false))) + { + poisonEntry.Type |= PoisonType.Hash; + var match = new PoisonMatch + { + Package = matchingCatalogedPackage.Path, + PackageId = matchingCatalogedPackage.Id, + PackageVersion = matchingCatalogedPackage.Version, + }; + poisonEntry.Matches.Add(match); + } + + // now extract and look for the marker file + if (ZipFileExtensions.Any(e => zipToCheck.ToLowerInvariant().EndsWith(e))) + { + ZipFile.ExtractToDirectory(zipToCheck, tempDir, true); + } + else if (TarFileExtensions.Any(e => zipToCheck.ToLowerInvariant().EndsWith(e))) + { + Directory.CreateDirectory(tempDir); + var psi = new ProcessStartInfo("tar", $"xf {zipToCheck} -C {tempDir}"); + Process.Start(psi).WaitForExit(); + } + else if (TarGzFileExtensions.Any(e => zipToCheck.ToLowerInvariant().EndsWith(e))) + { + Directory.CreateDirectory(tempDir); + var psi = new ProcessStartInfo("tar", $"xzf {zipToCheck} -C {tempDir}"); + Process.Start(psi).WaitForExit(); + } + else + { + throw new ArgumentOutOfRangeException($"Don't know how to decompress {zipToCheck}"); + } + + if (!string.IsNullOrWhiteSpace(markerFileName) && File.Exists(Path.Combine(tempDir, markerFileName))) + { + poisonEntry.Type |= PoisonType.NupkgFile; + } + + foreach (var child in Directory.EnumerateFiles(tempDir, "*", SearchOption.AllDirectories)) + { + // also add anything in this zip/package for checking + futureFilesToCheck.Enqueue(child); + } + + return poisonEntry.Type != PoisonType.None ? poisonEntry : null; + } + + private static IEnumerable ReadCatalog(string hashCatalogFilePath) + { + // catalog is optional, we can also just check assembly properties or nupkg marker files + if (string.IsNullOrWhiteSpace(hashCatalogFilePath)) + { + return Enumerable.Empty(); + } + + var doc = new XmlDocument(); + using (var stream = File.OpenRead(hashCatalogFilePath)) + { + doc.Load(stream); + } + var packages = new List(); + var catalog = doc.FirstChild; + foreach (XmlElement p in catalog.ChildNodes) + { + var package = new CatalogPackageEntry + { + Id = p.Attributes["Id"].Value, + Version = p.Attributes["Version"].Value, + OriginalHash = p.Attributes["OriginalHash"].Value.ToBytes(), + PoisonedHash = p.Attributes["PoisonedHash"]?.Value?.ToBytes(), + Path = p.Attributes["Path"].Value, + }; + packages.Add(package); + foreach (XmlNode f in p.ChildNodes) + { + var fEntry = new CatalogFileEntry + { + OriginalHash = f.Attributes["OriginalHash"].Value.ToBytes(), + PoisonedHash = f.Attributes["PoisonedHash"]?.Value?.ToBytes(), + Path = f.Attributes["Path"].Value, + }; + package.Files.Add(fEntry); + } + } + return packages; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/MarkAndCatalogPackages.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/MarkAndCatalogPackages.cs new file mode 100644 index 000000000..0ef1ed0af --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/MarkAndCatalogPackages.cs @@ -0,0 +1,218 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Mono.Cecil; +using Mono.Collections.Generic; +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Reflection; +using System.Runtime.CompilerServices; +using System.Security.Cryptography; +using System.Text; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + public class MarkAndCatalogPackages : Task + { + private const string CatalogElementName = "HashCatalog"; + private const string PoisonMarker = "POISONED by DotNetSourceBuild - Should not ship"; + + private readonly Type[] AssemblyPropertiesToReplace = new Type[] { + typeof(AssemblyProductAttribute), + typeof(AssemblyInformationalVersionAttribute), + typeof(AssemblyDescriptionAttribute), + typeof(AssemblyTitleAttribute) + }; + + /// + /// The name of the XML file to write the hash catalog out to, + /// for later checking build output against. This is optional - + /// if not used, assemblies will still be poisoned in their attributes. + /// + public string CatalogOutputFilePath { get; set; } + + /// + /// The name of the marker file to drop in the nupkgs. This can vary + /// with the packages, so you can use ".prebuilt" for one set of packages + /// and ".source-built" for another if you would like to tell the difference + /// between two sets of poisoned packages. + /// + public string MarkerFileName { get; set; } + + /// + /// The packages to poison and/or catalog: + /// %(Identity): Path to the nupkg. + /// + [Required] + public ITaskItem[] PackagesToMark { get; set; } + + /// + /// Use this directory instead of the system temp directory for staging. + /// Intended for Linux systems with limited /tmp space, like Azure VMs. + /// + public string OverrideTempPath { get; set; } + + public override bool Execute() + { + var tempDirName = Path.GetRandomFileName(); + if (!string.IsNullOrWhiteSpace(OverrideTempPath)) + { + Directory.CreateDirectory(OverrideTempPath); + } + var tempDir = Directory.CreateDirectory(Path.Combine(OverrideTempPath ?? Path.GetTempPath(), tempDirName)); + + var packageEntries = new List(); + + using (var sha = SHA256.Create()) + { + foreach (var p in PackagesToMark) + { + var packageEntry = new CatalogPackageEntry(); + packageEntries.Add(packageEntry); + packageEntry.Path = p.ItemSpec; + using (var stream = File.OpenRead(p.ItemSpec)) + { + packageEntry.OriginalHash = sha.ComputeHash(stream); + } + var packageIdentity = ReadNuGetPackageInfos.ReadIdentity(p.ItemSpec); + packageEntry.Id = packageIdentity.Id; + packageEntry.Version = packageIdentity.Version.OriginalVersion; + var packageTempPath = Path.Combine(tempDir.FullName, Path.GetFileName(p.ItemSpec)); + ZipFile.ExtractToDirectory(p.ItemSpec, packageTempPath, true); + + foreach (string f in Directory.EnumerateFiles(packageTempPath, "*", SearchOption.AllDirectories)) + { + // remove signatures so we don't later fail validation + if (Path.GetFileName(f) == ".signature.p7s") + { + File.Delete(f); + continue; + } + + var catalogFileEntry = new CatalogFileEntry(); + packageEntry.Files.Add(catalogFileEntry); + catalogFileEntry.Path = Utility.MakeRelativePath(f, packageTempPath); + AssemblyDefinition asm = null; + + // There seem to be some weird issues with using a file stream both for hashing and + // assembly loading, even closing it in between. Use a MemoryStream to avoid issues. + var memStream = new MemoryStream(); + using (var stream = File.OpenRead(f)) + { + stream.CopyTo(memStream); + } + + // First get the original hash of the file + memStream.Seek(0, SeekOrigin.Begin); + catalogFileEntry.OriginalHash = sha.ComputeHash(memStream); + + // Now try to read it as an assembly + memStream.Seek(0, SeekOrigin.Begin); + try + { + asm = AssemblyDefinition.ReadAssembly(memStream, new ReaderParameters(ReadingMode.Deferred)); + } + catch + { + // this is okay, it's not an assembly we can read + } + + // if we read it, now poison and write it back out + if (asm != null) + { + Poison(asm); + + try + { + // Cecil doesn't try to do some modifications until it writes out the file, + // and then throws after we've already truncated the file if it finds out it can't do them. + // Write to a memory stream first and then copy to the real stream if it suceeds. If it + // fails, we won't truncate the file and we will depend on hashes instead in that case. + using (var testMemStream = new MemoryStream()) + { + asm.Write(testMemStream); + testMemStream.Seek(0, SeekOrigin.Begin); + using (var stream = File.Open(f, FileMode.Create, FileAccess.ReadWrite)) + { + testMemStream.CopyTo(stream); + } + } + + // then get the hash of the now-poisoned file + using (var stream = File.OpenRead(f)) + { + catalogFileEntry.PoisonedHash = sha.ComputeHash(stream); + } + } + catch + { + // see above note in the try - this is okay. + } + } + } + + if (!string.IsNullOrWhiteSpace(MarkerFileName)) + { + var markerFilePath = Path.Combine(packageTempPath, MarkerFileName); + + if (File.Exists(markerFilePath)) + { + throw new ArgumentException($"Marker file name '{MarkerFileName}' is not sufficiently unique! Exists in '{p.ItemSpec}'.", nameof(MarkerFileName)); + } + + // mostly we just need to write something unique to this so it's not hashed as a matching file when we check it later. + // but it's also convenient to have the package catalog handy. + File.WriteAllText(markerFilePath, packageEntry.ToXml().ToString()); + } + + // create a temp file for this so if something goes wrong in the process we're not in too weird of a state + var poisonedPackageName = Path.GetFileName(p.ItemSpec) + ".poisoned"; + var poisonedPackagePath = Path.Combine(tempDir.FullName, poisonedPackageName); + ZipFile.CreateFromDirectory(packageTempPath, poisonedPackagePath); + + // Get the hash of the poisoned package (with poisoned marker file and poisoned assemblies inside) + using (var stream = File.OpenRead(poisonedPackagePath)) + { + packageEntry.PoisonedHash = sha.ComputeHash(stream); + } + File.Delete(p.ItemSpec); + File.Move(poisonedPackagePath, p.ItemSpec); + } + } + + // if we should write out the catalog, do that + if (!string.IsNullOrWhiteSpace(CatalogOutputFilePath)) + { + var outputFileDir = Path.GetDirectoryName(CatalogOutputFilePath); + if (!Directory.Exists(outputFileDir)) + { + Directory.CreateDirectory(outputFileDir); + } + File.WriteAllText(CatalogOutputFilePath, (new XElement("HashCatalog", + packageEntries.Select(p => p.ToXml()))).ToString()); + } + + tempDir.Delete(true); + return !Log.HasLoggedErrors; + } + + private void Poison(AssemblyDefinition asm) + { + foreach (var attr in asm.CustomAttributes) + { + if (this.AssemblyPropertiesToReplace.Any(p => p.Name == attr.AttributeType.Name)) + { + attr.ConstructorArguments.Clear(); + attr.ConstructorArguments.Add(new CustomAttributeArgument(asm.MainModule.TypeSystem.String, "POISONED by DotNetSourceBuild - Should not ship")); + } + } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj new file mode 100644 index 000000000..ab8eb402b --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection.csproj @@ -0,0 +1,35 @@ + + + + netcoreapp5.0 + true + $(LeakDetectionTasksBinDir) + + + + + 15.7.179 + + + 15.7.179 + + + Configuration=netstandard_Debug + Configuration=netstandard_Release + {D68133BD-1E63-496E-9EDE-4FBDBF77B486} + Mono.Cecil + + + Configuration=netstandard_Debug + Configuration=netstandard_Release + {D68133BD-1E63-496E-9EDE-4FBDBF77B486} + Mono.Cecil + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/NuGet.Config b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/NuGet.Config new file mode 100644 index 000000000..14757d86b --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/NuGet.Config @@ -0,0 +1,7 @@ + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonMatch.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonMatch.cs new file mode 100644 index 000000000..0191b3a26 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonMatch.cs @@ -0,0 +1,28 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Text; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + internal class PoisonMatch + { + const string ElementName = "Match"; + + internal string Package { get; set; } + internal string File { get; set; } + internal string PackageId { get; set; } + internal string PackageVersion { get; set; } + + public XElement ToXml() => new XElement(ElementName, + Package == null ? null : new XAttribute(nameof(Package), Package), + PackageId == null ? null : new XAttribute(nameof(PackageId), PackageId), + PackageVersion == null ? null : new XAttribute(nameof(PackageVersion), PackageVersion), + File == null ? null: new XAttribute(nameof(File), File) + ); + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonType.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonType.cs new file mode 100644 index 000000000..de5c35961 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonType.cs @@ -0,0 +1,15 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + [Flags] + internal enum PoisonType + { + None = 0, + Hash = 1, + AssemblyAttribute = 2, + NupkgFile = 4, + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonedFileEntry.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonedFileEntry.cs new file mode 100644 index 000000000..1d98fd8cb --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/PoisonedFileEntry.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + internal class PoisonedFileEntry + { + const string ElementName = "File"; + + internal byte[] Hash { get; set; } + internal string Path { get; set; } + internal PoisonType Type { get; set; } + internal List Matches { get; } + + internal PoisonedFileEntry() + { + this.Matches = new List(); + } + + public XElement ToXml() => this.ToXml(ElementName); + + protected XElement ToXml(string myElementName) => new XElement(myElementName, + new XAttribute(nameof(Path), Path), + new XElement(nameof(Hash), Hash.ToHexString()), + new XElement(nameof(Type), Type.ToString()), + Matches.Select(m => m.ToXml()) + ); + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Utility.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Utility.cs new file mode 100644 index 000000000..48df5d261 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/Utility.cs @@ -0,0 +1,43 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Microsoft.DotNet.SourceBuild.Tasks.LeakDetection +{ + internal static class Utility + { + internal static string ToHexString(this byte[] bytes) + { + var sb = new StringBuilder(); + foreach (var b in bytes) + { + sb.Append(b.ToString("x2")); + } + return sb.ToString(); + } + + internal static byte[] ToBytes(this string hex) + { + var bytes = new List(); + for (var i = 0; i < hex.Length; i += 2) + { + bytes.Add(Convert.ToByte(hex.Substring(i, 2), 16)); + } + return bytes.ToArray(); + } + + internal static string MakeRelativePath(string filePath, string relativeTo) + { + // Uri.MakeRelativeUri requires the last slash + if (!relativeTo.EndsWith("/") && !relativeTo.EndsWith("\\")) + { + relativeTo += Path.DirectorySeparatorChar; + } + + var uri = new Uri(filePath); + var relativeToUri = new Uri(relativeTo); + return relativeToUri.MakeRelativeUri(uri).ToString(); + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/_._ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.LeakDetection/_._ new file mode 100644 index 000000000..e69de29bb diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AddRidToRuntimeJson.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AddRidToRuntimeJson.cs new file mode 100644 index 000000000..5842d0d82 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AddRidToRuntimeJson.cs @@ -0,0 +1,94 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Text; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using System.Collections.Generic; +using System.Linq; +using NuGet.Versioning; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class AddRidToRuntimeJson:Task + { + /// + /// [OS name].[version]-[architecture] + /// + [Required] + public string Rid { get; set; } + + [Required] + public string RuntimeJson { get; set; } + + private string runtimesIdentifier = "runtimes"; + + public override bool Execute() + { + string[] ridParts = Rid.Split('-'); + string osNameAndVersion = ridParts[0]; + string[] osParts = osNameAndVersion.Split(new char[] { '.' }, 2); + + if (ridParts.Length < 1 || osParts.Length < 2) + { + throw new System.InvalidOperationException($"Unknown rid format {Rid}."); + } + + // Acquire Rid parts: + // osName + // version + // arch + string arch = ridParts[1]; + string osName = osParts[0]; + string version = osParts[1]; + + JObject projectRoot = ReadProject(RuntimeJson); + + if (projectRoot.SelectToken($"{runtimesIdentifier}.{osName}") == null) + { + AddRidToRuntimeGraph(projectRoot, osName, "linux"); + AddRidToRuntimeGraph(projectRoot, $"{osName}-{arch}", osName, $"linux-{arch}"); + } + if(projectRoot.SelectToken($"{runtimesIdentifier}.{osName}.{version}") == null) + { + AddRidToRuntimeGraph(projectRoot, $"{osName}.{version}", osName); + AddRidToRuntimeGraph(projectRoot, $"{osName}.{version}-{arch}", $"{osName}.{version}", $"{osName}-{arch}"); + } + + WriteProject(projectRoot, RuntimeJson); + return true; + } + + private void AddRidToRuntimeGraph(JObject projectRoot, string name, params string[] imports) + { + projectRoot[runtimesIdentifier][name] = new JObject(new JProperty("#import", new JArray(imports))); + } + + private static JObject ReadProject(string projectJsonPath) + { + using (TextReader projectFileReader = File.OpenText(projectJsonPath)) + { + var projectJsonReader = new JsonTextReader(projectFileReader); + var serializer = new JsonSerializer(); + return serializer.Deserialize(projectJsonReader); + } + } + private static void WriteProject(JObject projectRoot, string projectJsonPath) + { + string projectJson = JsonConvert.SerializeObject(projectRoot, Formatting.Indented) + Environment.NewLine; + + if (!File.Exists(projectJsonPath) || !projectJson.Equals(File.ReadAllText(projectJsonPath))) + { + Directory.CreateDirectory(Path.GetDirectoryName(projectJsonPath)); + File.WriteAllText(projectJsonPath, projectJson); + } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AddSourceToNuGetConfig.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AddSourceToNuGetConfig.cs new file mode 100755 index 000000000..a00a8a550 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AddSourceToNuGetConfig.cs @@ -0,0 +1,63 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + + +namespace Microsoft.DotNet.Build.Tasks +{ + /* + * This task adds a source to a well-formed NuGet.Config file. If a source with `SourceName` is already present, then + * the path of the source is changed. Otherwise, the source is added as the first source in the list, after any clear + * elements (if present). + */ + public class AddSourceToNuGetConfig : Task + { + [Required] + public string NuGetConfigFile { get; set; } + + [Required] + public string SourceName { get; set; } + + [Required] + public string SourcePath { get; set; } + + public override bool Execute() + { + XDocument d = XDocument.Load(NuGetConfigFile); + XElement packageSourcesElement = d.Root.Descendants().First(e => e.Name == "packageSources"); + XElement toAdd = new XElement("add", new XAttribute("key", SourceName), new XAttribute("value", SourcePath)); + XElement clearTag = new XElement("clear"); + + XElement exisitingSourceBuildElement = packageSourcesElement.Descendants().FirstOrDefault(e => e.Name == "add" && e.Attribute(XName.Get("key")).Value == SourceName); + XElement lastClearElement = packageSourcesElement.Descendants().LastOrDefault(e => e.Name == "clear"); + + if (exisitingSourceBuildElement != null) + { + exisitingSourceBuildElement.ReplaceWith(toAdd); + } + else if (lastClearElement != null) + { + lastClearElement.AddAfterSelf(toAdd); + } + else + { + packageSourcesElement.AddFirst(toAdd); + packageSourcesElement.AddFirst(clearTag); + } + + using (FileStream fs = new FileStream(NuGetConfigFile, FileMode.Create, FileAccess.ReadWrite)) + { + d.Save(fs); + } + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AzureConnectionStringBuildTask.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AzureConnectionStringBuildTask.cs new file mode 100644 index 000000000..b717d3450 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AzureConnectionStringBuildTask.cs @@ -0,0 +1,61 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Utilities; +using System.Text.RegularExpressions; + +namespace Microsoft.DotNet.Build.Tasks +{ + public abstract class AzureConnectionStringBuildTask : Task + { + /// + /// Azure Storage account connection string. Supersedes Account Key / Name. + /// Will cause errors if both are set. + /// + public string ConnectionString { get; set; } + + /// + /// The Azure account key used when creating the connection string. + /// When we fully deprecate these, can just make them get; only. + /// + public string AccountKey { get; set; } + + /// + /// The Azure account name used when creating the connection string. + /// When we fully deprecate these, can just make them get; only. + /// + public string AccountName { get; set; } + + public void ParseConnectionString() + { + if (!string.IsNullOrEmpty(ConnectionString)) + { + if (!(string.IsNullOrEmpty(AccountKey) && string.IsNullOrEmpty(AccountName))) + { + Log.LogError("If the ConnectionString property is set, you must not provide AccountKey / AccountName. These values will be deprecated in the future."); + } + else + { + Regex storageConnectionStringRegex = new Regex("AccountName=(?.+?);AccountKey=(?.+?);"); + + MatchCollection matches = storageConnectionStringRegex.Matches(ConnectionString); + if (matches.Count > 0) + { + // When we deprecate this format, we'll want to demote these to private + AccountName = matches[0].Groups["name"].Value; + AccountKey = matches[0].Groups["key"].Value; + } + else + { + Log.LogError("Error parsing connection string. Please review its value."); + } + } + } + else if (string.IsNullOrEmpty(AccountKey) || string.IsNullOrEmpty(AccountName)) + { + Log.LogError("Error, must provide either ConnectionString or AccountName with AccountKey"); + } + } + } +} \ No newline at end of file diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AzureHelper.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AzureHelper.cs new file mode 100644 index 000000000..ac08cfb3a --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/AzureHelper.cs @@ -0,0 +1,461 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Security.Cryptography; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; + +namespace Microsoft.DotNet.Build.Tasks +{ + public static class AzureHelper + { + /// + /// The storage api version. + /// + public static readonly string StorageApiVersion = "2015-04-05"; + public const string DateHeaderString = "x-ms-date"; + public const string VersionHeaderString = "x-ms-version"; + public const string AuthorizationHeaderString = "Authorization"; + public const string CacheControlString = "x-ms-blob-cache-control"; + public const string ContentTypeString = "x-ms-blob-content-type"; + + public enum SasAccessType + { + Read, + Write, + }; + + public static string AuthorizationHeader( + string storageAccount, + string storageKey, + string method, + DateTime now, + HttpRequestMessage request, + string ifMatch = "", + string contentMD5 = "", + string size = "", + string contentType = "") + { + string stringToSign = string.Format( + "{0}\n\n\n{1}\n{5}\n{6}\n\n\n{2}\n\n\n\n{3}{4}", + method, + (size == string.Empty) ? string.Empty : size, + ifMatch, + GetCanonicalizedHeaders(request), + GetCanonicalizedResource(request.RequestUri, storageAccount), + contentMD5, + contentType); + byte[] signatureBytes = Encoding.UTF8.GetBytes(stringToSign); + string authorizationHeader; + using (HMACSHA256 hmacsha256 = new HMACSHA256(Convert.FromBase64String(storageKey))) + { + authorizationHeader = "SharedKey " + storageAccount + ":" + + Convert.ToBase64String(hmacsha256.ComputeHash(signatureBytes)); + } + + return authorizationHeader; + } + + public static string CreateContainerSasToken( + string accountName, + string containerName, + string key, + SasAccessType accessType, + int validityTimeInDays) + { + string signedPermissions = string.Empty; + switch (accessType) + { + case SasAccessType.Read: + signedPermissions = "r"; + break; + case SasAccessType.Write: + signedPermissions = "wdl"; + break; + default: + throw new ArgumentOutOfRangeException(nameof(accessType), accessType, "Unrecognized value"); + } + + string signedStart = DateTime.UtcNow.ToString("O"); + string signedExpiry = DateTime.UtcNow.AddDays(validityTimeInDays).ToString("O"); + string canonicalizedResource = "/blob/" + accountName + "/" + containerName; + string signedIdentifier = string.Empty; + string signedVersion = StorageApiVersion; + + string stringToSign = ConstructServiceStringToSign( + signedPermissions, + signedVersion, + signedExpiry, + canonicalizedResource, + signedIdentifier, + signedStart); + + byte[] signatureBytes = Encoding.UTF8.GetBytes(stringToSign); + string signature; + using (HMACSHA256 hmacSha256 = new HMACSHA256(Convert.FromBase64String(key))) + { + signature = Convert.ToBase64String(hmacSha256.ComputeHash(signatureBytes)); + } + + string sasToken = string.Format( + "?sv={0}&sr={1}&sig={2}&st={3}&se={4}&sp={5}", + WebUtility.UrlEncode(signedVersion), + WebUtility.UrlEncode("c"), + WebUtility.UrlEncode(signature), + WebUtility.UrlEncode(signedStart), + WebUtility.UrlEncode(signedExpiry), + WebUtility.UrlEncode(signedPermissions)); + + return sasToken; + } + + public static string GetCanonicalizedHeaders(HttpRequestMessage request) + { + StringBuilder sb = new StringBuilder(); + List headerNameList = (from headerName in request.Headers + where + headerName.Key.ToLowerInvariant() + .StartsWith("x-ms-", StringComparison.Ordinal) + select headerName.Key.ToLowerInvariant()).ToList(); + headerNameList.Sort(); + foreach (string headerName in headerNameList) + { + StringBuilder builder = new StringBuilder(headerName); + string separator = ":"; + foreach (string headerValue in GetHeaderValues(request.Headers, headerName)) + { + string trimmedValue = headerValue.Replace("\r\n", string.Empty); + builder.Append(separator); + builder.Append(trimmedValue); + separator = ","; + } + + sb.Append(builder); + sb.Append("\n"); + } + + return sb.ToString(); + } + + public static string GetCanonicalizedResource(Uri address, string accountName) + { + StringBuilder str = new StringBuilder(); + StringBuilder builder = new StringBuilder("/"); + builder.Append(accountName); + builder.Append(address.AbsolutePath); + str.Append(builder); + Dictionary> queryKeyValues = ExtractQueryKeyValues(address); + Dictionary> dictionary = GetCommaSeparatedList(queryKeyValues); + + foreach (KeyValuePair> pair in dictionary.OrderBy(p => p.Key)) + { + StringBuilder stringBuilder = new StringBuilder(string.Empty); + stringBuilder.Append(pair.Key + ":"); + string commaList = string.Join(",", pair.Value); + stringBuilder.Append(commaList); + str.Append("\n"); + str.Append(stringBuilder); + } + + return str.ToString(); + } + + public static List GetHeaderValues(HttpRequestHeaders headers, string headerName) + { + List list = new List(); + IEnumerable values; + headers.TryGetValues(headerName, out values); + if (values != null) + { + list.Add((values.FirstOrDefault() ?? string.Empty).TrimStart(null)); + } + + return list; + } + + private static bool IsWithinRetryRange(HttpStatusCode statusCode) + { + // Retry on http client and server error codes (4xx - 5xx) as well as redirect + + var rawStatus = (int)statusCode; + if (rawStatus == 302) + return true; + else if (rawStatus >= 400 && rawStatus <= 599) + return true; + else + return false; + } + + public static async Task RequestWithRetry(TaskLoggingHelper loggingHelper, HttpClient client, + Func createRequest, Func validationCallback = null, int retryCount = 5, + int retryDelaySeconds = 5) + { + if (loggingHelper == null) + throw new ArgumentNullException(nameof(loggingHelper)); + if (client == null) + throw new ArgumentNullException(nameof(client)); + if (createRequest == null) + throw new ArgumentNullException(nameof(createRequest)); + if (retryCount < 1) + throw new ArgumentException(nameof(retryCount)); + if (retryDelaySeconds < 1) + throw new ArgumentException(nameof(retryDelaySeconds)); + + int retries = 0; + HttpResponseMessage response = null; + + // add a bit of randomness to the retry delay + var rng = new Random(); + + while (retries < retryCount) + { + if (retries > 0) + { + if (response != null) + { + response.Dispose(); + response = null; + } + + int delay = retryDelaySeconds * retries * rng.Next(1, 5); + loggingHelper.LogMessage(MessageImportance.Low, "Waiting {0} seconds before retry", delay); + await System.Threading.Tasks.Task.Delay(delay * 1000); + } + + try + { + using (var request = createRequest()) + response = await client.SendAsync(request); + } + catch (Exception e) + { + loggingHelper.LogWarningFromException(e, true); + + // if this is the final iteration let the exception bubble up + if (retries + 1 == retryCount) + throw; + } + + // response can be null if we fail to send the request + if (response != null) + { + if (validationCallback == null) + { + // check if the response code is within the range of failures + if (!IsWithinRetryRange(response.StatusCode)) + { + return response; + } + } + else + { + bool isSuccess = validationCallback(response); + if (!isSuccess) + { + loggingHelper.LogMessage("Validation callback returned retry for status code {0}", response.StatusCode); + } + else + { + loggingHelper.LogMessage("Validation callback returned success for status code {0}", response.StatusCode); + return response; + } + } + } + + ++retries; + } + + // retry count exceeded + loggingHelper.LogWarning("Retry count {0} exceeded", retryCount); + + // set some default values in case response is null + var statusCode = "None"; + var contentStr = "Null"; + if (response != null) + { + statusCode = response.StatusCode.ToString(); + contentStr = await response.Content.ReadAsStringAsync(); + response.Dispose(); + } + + throw new HttpRequestException($"Request {createRequest().RequestUri} failed with status {statusCode}. Response : {contentStr}"); + } + + private static string ConstructServiceStringToSign( + string signedPermissions, + string signedVersion, + string signedExpiry, + string canonicalizedResource, + string signedIdentifier, + string signedStart, + string signedIP = "", + string signedProtocol = "", + string rscc = "", + string rscd = "", + string rsce = "", + string rscl = "", + string rsct = "") + { + // constructing string to sign based on spec in https://msdn.microsoft.com/en-us/library/azure/dn140255.aspx + var stringToSign = string.Join( + "\n", + signedPermissions, + signedStart, + signedExpiry, + canonicalizedResource, + signedIdentifier, + signedIP, + signedProtocol, + signedVersion, + rscc, + rscd, + rsce, + rscl, + rsct); + return stringToSign; + } + + private static Dictionary> ExtractQueryKeyValues(Uri address) + { + Dictionary> values = new Dictionary>(); + //Decode this to allow the regex to pull out the correct groups for signing + address = new Uri(WebUtility.UrlDecode(address.ToString())); + Regex newreg = new Regex(@"(?:\?|&)([^=]+)=([^&]+)"); + MatchCollection matches = newreg.Matches(address.Query); + foreach (Match match in matches) + { + string key, value; + if (!string.IsNullOrEmpty(match.Groups[1].Value)) + { + key = match.Groups[1].Value; + value = match.Groups[2].Value; + } + else + { + key = match.Groups[3].Value; + value = match.Groups[4].Value; + } + + HashSet setOfValues; + if (values.TryGetValue(key, out setOfValues)) + { + setOfValues.Add(value); + } + else + { + HashSet newSet = new HashSet { value }; + values.Add(key, newSet); + } + } + + return values; + } + + private static Dictionary> GetCommaSeparatedList( + Dictionary> queryKeyValues) + { + Dictionary> dictionary = new Dictionary>(); + + foreach (string queryKeys in queryKeyValues.Keys) + { + HashSet setOfValues; + queryKeyValues.TryGetValue(queryKeys, out setOfValues); + List list = new List(); + list.AddRange(setOfValues); + list.Sort(); + string commaSeparatedValues = string.Join(",", list); + string key = queryKeys.ToLowerInvariant(); + HashSet setOfValues2; + if (dictionary.TryGetValue(key, out setOfValues2)) + { + setOfValues2.Add(commaSeparatedValues); + } + else + { + HashSet newSet = new HashSet { commaSeparatedValues }; + dictionary.Add(key, newSet); + } + } + + return dictionary; + } + + public static Func RequestMessage(string method, string url, string accountName, string accountKey, List> additionalHeaders = null, string body = null) + { + Func requestFunc = () => + { + HttpMethod httpMethod = HttpMethod.Get; + if (method == "PUT") + { + httpMethod = HttpMethod.Put; + } + else if (method == "DELETE") + { + httpMethod = HttpMethod.Delete; + } + DateTime dateTime = DateTime.UtcNow; + var request = new HttpRequestMessage(httpMethod, url); + request.Headers.Add(AzureHelper.DateHeaderString, dateTime.ToString("R", CultureInfo.InvariantCulture)); + request.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); + if (additionalHeaders != null) + { + foreach (Tuple additionalHeader in additionalHeaders) + { + request.Headers.Add(additionalHeader.Item1, additionalHeader.Item2); + } + } + if (body != null) + { + request.Content = new StringContent(body); + request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( + accountName, + accountKey, + method, + dateTime, + request, + "", + "", + request.Content.Headers.ContentLength.ToString(), + request.Content.Headers.ContentType.ToString())); + } + else + { + request.Headers.Add(AzureHelper.AuthorizationHeaderString, AzureHelper.AuthorizationHeader( + accountName, + accountKey, + method, + dateTime, + request)); + } + return request; + }; + return requestFunc; + } + + public static string GetRootRestUrl(string accountName) + { + return $"https://{accountName}.blob.core.windows.net"; + } + + public static string GetContainerRestUrl(string accountName, string containerName) + { + return $"{GetRootRestUrl(accountName)}/{containerName}"; + } + + public static string GetBlobRestUrl(string accountName, string containerName, string blob) + { + return $"{GetContainerRestUrl(accountName, containerName)}/{blob}"; + } + } +} \ No newline at end of file diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/BuildTask.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/BuildTask.cs new file mode 100644 index 000000000..f4fd687e9 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/BuildTask.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + public abstract partial class BuildTask : ITask + { + private TaskLoggingHelper _log = null; + + internal TaskLoggingHelper Log + { + get { return _log ?? (_log = new TaskLoggingHelper(this)); } + } + + public BuildTask() + { + } + + public IBuildEngine BuildEngine + { + get; + set; + } + + public ITaskHost HostObject + { + get; + set; + } + + public abstract bool Execute(); + } +} \ No newline at end of file diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/DownloadFileSB.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/DownloadFileSB.cs new file mode 100644 index 000000000..680327baf --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/DownloadFileSB.cs @@ -0,0 +1,409 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +// This task is sourced from https://github.com/microsoft/msbuild/blob/04e508c36f9c1fe826264aef7c26ffb8f16e9bdc/src/Tasks/DownloadFile.cs +// Contains further modifications in followup commits. +// It alleviates the problem of time outs on DownloadFile Task. We are not the version of msbuild that has this fix, hence we have to locally +// build it to get rid of the issue. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Microsoft.DotNet.Build.Tasks; +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.Build.Tasks +{ + /// + /// Represents a task that can download a file. + /// + public sealed class DownloadFileSB : BuildTask, ICancelableTask + { + private readonly CancellationTokenSource _cancellationTokenSource = new CancellationTokenSource(); + + /// + /// Gets or sets an optional filename for the destination file. By default, the filename is derived from the if possible. + /// + public ITaskItem DestinationFileName { get; set; } + + /// + /// Gets or sets a that specifies the destination folder to download the file to. + /// + [Required] + public ITaskItem DestinationFolder { get; set; } + + /// + /// Gets or sets a that contains details about the downloaded file. + /// + [Output] + public ITaskItem DownloadedFile { get; set; } + + /// + /// Gets or sets an optional number of times to retry if possible. + /// + public int Retries { get; set; } + + /// + /// Gets or sets the number of milliseconds to wait before retrying. + /// + public int RetryDelayMilliseconds { get; set; } = 5 * 1000; + + /// + /// Gets or sets an optional value indicating whether or not the download should be skipped if the file is up-to-date. + /// + public bool SkipUnchangedFiles { get; set; } = true; + + /// + /// Gets or sets the URL to download. + /// + [Required] + public string SourceUrl { get; set; } + + /// + /// Gets or sets the timeout for a successful download. If exceeded, the download continues + /// for another two timeout durations before failing. This makes it sometimes possible to + /// determine whether the timeout is just a little too short, or if the download would never + /// have finished. + /// + public string TimeoutSeconds { get; set; } + + /// + /// Gets or sets a to use. This is used by unit tests to mock a connection to a remote server. + /// + internal HttpMessageHandler HttpMessageHandler { get; set; } + + /// + public void Cancel() + { + _cancellationTokenSource.Cancel(); + } + + public override bool Execute() + { + return ExecuteAsync().GetAwaiter().GetResult(); + } + + private async Task ExecuteAsync() + { + if (!Uri.TryCreate(SourceUrl, UriKind.Absolute, out Uri uri)) + { + Log.LogError($"DownloadFileSB.ErrorInvalidUrl {SourceUrl}"); + return false; + } + + int retryAttemptCount = 0; + + CancellationToken cancellationToken = _cancellationTokenSource.Token; + + var startTime = DateTime.UtcNow; + + // Use the same API for the "success timeout" and the "would it ever succeed" timeout. + var timeout = TimeSpan.Zero; + var successCancellationTokenSource = new CancellationTokenSource(); + + if (double.TryParse(TimeoutSeconds, out double timeoutSeconds)) + { + timeout = TimeSpan.FromSeconds(timeoutSeconds); + Log.LogMessage(MessageImportance.High, $"DownloadFileSB timeout set to {timeout}"); + + successCancellationTokenSource.CancelAfter(timeout); + _cancellationTokenSource.CancelAfter((int)(timeout.TotalMilliseconds * 3)); + } + + while (true) + { + try + { + await DownloadAsync(uri, cancellationToken); + break; + } + catch (OperationCanceledException e) when (e.CancellationToken == cancellationToken) + { + // This task is being cancelled. Exit the loop. + break; + } + catch (Exception e) + { + bool canRetry = IsRetriable(e, out Exception actualException) && retryAttemptCount++ < Retries; + + if (canRetry) + { + Log.LogWarning($"DownloadFileSB.Retrying {SourceUrl} {retryAttemptCount + 1} {RetryDelayMilliseconds} {actualException}"); + + try + { + await Task.Delay(RetryDelayMilliseconds, cancellationToken).ConfigureAwait(false); + } + catch (OperationCanceledException delayException) when (delayException.CancellationToken == cancellationToken) + { + // This task is being cancelled, exit the loop + break; + } + } + else + { + Log.LogError($"DownloadFileSB.ErrorDownloading {SourceUrl} {actualException}"); + break; + } + } + } + + var finishTime = DateTime.UtcNow; + + if (successCancellationTokenSource.IsCancellationRequested) + { + string error = $"{TimeoutSeconds} second timeout exceeded"; + + if (!_cancellationTokenSource.IsCancellationRequested) + { + error += + $", but download completed after {finishTime - startTime}. " + + $"Try increasing timeout from {TimeoutSeconds} if this is acceptable."; + } + else + { + error += + $", and didn't complete within leeway after {finishTime - startTime}. " + + $"The download was likely never going to terminate. Investigate logs and " + + $"add additional logging if necessary."; + } + + Log.LogError(error); + } + else + { + Log.LogMessage( + MessageImportance.High, + $"DownloadFileSB.Downloading Complete! Elapsed: {finishTime - startTime}"); + } + + return !_cancellationTokenSource.IsCancellationRequested && !Log.HasLoggedErrors; + } + + /// + /// Attempts to download the file. + /// + /// The parsed of the request. + private async Task DownloadAsync(Uri uri, CancellationToken cancellationToken) + { + // The main reason to use HttpClient vs WebClient is because we can pass a message handler for unit tests to mock + using (var client = new HttpClient(HttpMessageHandler ?? new HttpClientHandler(), disposeHandler: true)) + { + // Only get the response without downloading the file so we can determine if the file is already up-to-date + using (HttpResponseMessage response = await client.GetAsync(uri, HttpCompletionOption.ResponseHeadersRead, cancellationToken).ConfigureAwait(false)) + { + try + { + response.EnsureSuccessStatusCode(); + } + catch (HttpRequestException e) + { + // HttpRequestException does not have the status code so its wrapped and thrown here so that later on we can determine + // if a retry is possible based on the status code + throw new CustomHttpRequestException(e.Message, e.InnerException, response.StatusCode); + } + + if (!TryGetFileName(response, out string filename)) + { + Log.LogError($"DownloadFileSB.ErrorUnknownFileName {SourceUrl} {nameof(DestinationFileName)}"); + return; + } + + DirectoryInfo destinationDirectory = Directory.CreateDirectory(DestinationFolder.ItemSpec); + + var destinationFile = new FileInfo(Path.Combine(destinationDirectory.FullName, filename)); + + // The file is considered up-to-date if its the same length. This could be inaccurate, we can consider alternatives in the future + if (ShouldSkip(response, destinationFile)) + { + Log.LogMessage(MessageImportance.Normal, $"DownloadFileSB.DidNotDownloadBecauseOfFileMatch {SourceUrl}", destinationFile.FullName, nameof(SkipUnchangedFiles), "true"); + + DownloadedFile = new TaskItem(destinationFile.FullName); + + return; + } + + var progressMonitorCancellationTokenSource = new CancellationTokenSource(); + CancellationToken progressMonitorToken = progressMonitorCancellationTokenSource.Token; + + try + { + cancellationToken.ThrowIfCancellationRequested(); + + var startTime = DateTime.UtcNow; + + var progressMonitor = Task.Run( + async () => + { + while (!progressMonitorToken.IsCancellationRequested) + { + destinationFile.Refresh(); + if (destinationFile.Exists) + { + long current = destinationFile.Length; + long total = response.Content.Headers.ContentLength ?? 1; + var elapsed = DateTime.UtcNow - startTime; + double kbytesPerSecond = current / elapsed.TotalSeconds / 1000.0; + + Log.LogMessage( + MessageImportance.High, + $"Progress... {elapsed}, " + + $"current file size {current / (double)total:00.0%} " + + $"({destinationFile.Length:#,0} / {total:#,0}) " + + $"~ {kbytesPerSecond:#,0.00} kB/s"); + } + await Task.Delay(TimeSpan.FromSeconds(5), progressMonitorToken); + } + }, + progressMonitorToken) + .ConfigureAwait(false); + + using (var target = new FileStream(destinationFile.FullName, FileMode.Create, FileAccess.Write, FileShare.None)) + { + Log.LogMessage( + MessageImportance.High, + $"DownloadFileSB.Downloading {SourceUrl} to " + + $"{destinationFile.FullName}"); + + Log.LogMessage( MessageImportance.Low, $"All response headers:\n{response.Headers}"); + Log.LogMessage( MessageImportance.Low, $"All content headers:\n{response.Content.Headers}"); + + using (Stream responseStream = await response.Content.ReadAsStreamAsync().ConfigureAwait(false)) + { + await responseStream.CopyToAsync(target, 1024, cancellationToken).ConfigureAwait(false); + } + + Log.LogMessage(MessageImportance.High, $"DownloadFileSB.StreamCopyComplete {SourceUrl}"); + + DownloadedFile = new TaskItem(destinationFile.FullName); + } + } + finally + { + if (DownloadedFile == null) + { + // Delete the file if anything goes wrong during download. This could be destructive but we don't want to leave + // partially downloaded files on disk either. Alternatively we could download to a temporary location and copy + // on success but we are concerned about the added I/O + destinationFile.Delete(); + } + + progressMonitorCancellationTokenSource.Cancel(); + } + } + } + } + + /// + /// Determines if the specified exception is considered retriable. + /// + /// The originally thrown exception. + /// The actual exception to be used for logging errors. + /// true if the exception is retriable, otherwise false. + private static bool IsRetriable(Exception exception, out Exception actualException) + { + actualException = exception; + + // Get aggregate inner exception + if (actualException is AggregateException aggregateException && aggregateException.InnerException != null) + { + actualException = aggregateException.InnerException; + } + + // Some HttpRequestException have an inner exception that has the real error + if (actualException is HttpRequestException httpRequestException && httpRequestException.InnerException != null) + { + actualException = httpRequestException.InnerException; + + // An IOException inside of a HttpRequestException means that something went wrong while downloading + if (actualException is IOException) + { + return true; + } + } + + if (actualException is CustomHttpRequestException customHttpRequestException) + { + // A wrapped CustomHttpRequestException has the status code from the error + switch (customHttpRequestException.StatusCode) + { + case HttpStatusCode.InternalServerError: + case HttpStatusCode.RequestTimeout: + return true; + } + } + + if (actualException is WebException webException) + { + // WebException is thrown when accessing the Content of the response + switch (webException.Status) + { + // Don't retry on anything that cannot be compensated for + case WebExceptionStatus.TrustFailure: + case WebExceptionStatus.MessageLengthLimitExceeded: + case WebExceptionStatus.RequestProhibitedByCachePolicy: + case WebExceptionStatus.RequestProhibitedByProxy: + return false; + + default: + // Retry on all other WebExceptions + return true; + } + } + + return false; + } + + /// + /// Attempts to get the file name to use when downloading the file. + /// + /// The with information about the response. + /// Receives the name of the file. + /// true if a file name could be determined, otherwise false. + private bool TryGetFileName(HttpResponseMessage response, out string filename) + { + if (response == null) + { + throw new ArgumentNullException(nameof(response)); + } + + // Not all URIs contain a file name so users will have to specify one + // Example: http://www.download.com/file/1/ + + filename = !String.IsNullOrWhiteSpace(DestinationFileName?.ItemSpec) + ? DestinationFileName.ItemSpec // Get the file name from what the user specified + : response.Content?.Headers?.ContentDisposition?.FileName // Attempt to get the file name from the content-disposition header value + ?? Path.GetFileName(response.RequestMessage.RequestUri.LocalPath); // Otherwise attempt to get a file name from the URI + + return !String.IsNullOrWhiteSpace(filename); + } + + /// + /// Represents a wrapper around the that also contains the . + /// + private sealed class CustomHttpRequestException : HttpRequestException + { + public CustomHttpRequestException(string message, Exception inner, HttpStatusCode statusCode) + : base(message, inner) + { + StatusCode = statusCode; + } + + public HttpStatusCode StatusCode { get; } + } + + private bool ShouldSkip(HttpResponseMessage response, FileInfo destinationFile) + { + return SkipUnchangedFiles + && destinationFile.Exists + && destinationFile.Length == response.Content.Headers.ContentLength + && response.Content.Headers.LastModified.HasValue + && destinationFile.LastWriteTimeUtc > response.Content.Headers.LastModified.Value.UtcDateTime; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/EnumerableExtensions.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/EnumerableExtensions.cs new file mode 100644 index 000000000..74102cff0 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/EnumerableExtensions.cs @@ -0,0 +1,17 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks +{ + internal static class EnumerableExtensions + { + public static IEnumerable NullAsEmpty(this IEnumerable source) + { + return source ?? Enumerable.Empty(); + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/FixPathSeparator.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/FixPathSeparator.cs new file mode 100755 index 000000000..263db19cf --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/FixPathSeparator.cs @@ -0,0 +1,86 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + /* + * This task replaces both types of path separators ('/' and '\') with the separator for the current + * platform. This workaround a NuGet issue where `nuget pack` does not translate path separators causing + * packages that don't appear to have the right assets in them. + */ + public class FixPathSeparator : Task + { + [Required] + public ITaskItem[] NuSpecFiles { get; set; } + + public override bool Execute() + { + foreach (ITaskItem item in NuSpecFiles) + { + string pathToNuSpec = item.GetMetadata("FullPath"); + + XDocument doc = XDocument.Load(pathToNuSpec); + + XElement contentFilesElement = doc.ElementIgnoringNamespace("package").ElementIgnoringNamespace("metadata").ElementIgnoringNamespace("contentFiles"); + XElement filesElement = doc.ElementIgnoringNamespace("package").ElementIgnoringNamespace("files"); + + if (contentFilesElement != null) + { + foreach (XElement element in contentFilesElement.ElementsIgnroingNamespace("files")) + { + UpdateDirectorySeperatorInAttribute(element, "include"); + UpdateDirectorySeperatorInAttribute(element, "exclude"); + } + } + + if (filesElement != null) + { + foreach (XElement element in filesElement.ElementsIgnroingNamespace("file")) + { + UpdateDirectorySeperatorInAttribute(element, "src"); + UpdateDirectorySeperatorInAttribute(element, "target"); + UpdateDirectorySeperatorInAttribute(element, "exclude"); + } + } + + using (FileStream fs = File.Open(pathToNuSpec, FileMode.Truncate)) + { + doc.Save(fs); + } + } + + return true; + } + + private static void UpdateDirectorySeperatorInAttribute(XElement element, XName name) + { + XAttribute attribute = element.Attribute(name); + + if (attribute != null) + { + element.SetAttributeValue(name, attribute.Value.Replace('/', Path.DirectorySeparatorChar).Replace('\\', Path.DirectorySeparatorChar)); + } + } + } + + static class XContainerExtensions + { + public static IEnumerable ElementsIgnroingNamespace(this XContainer container, XName elementName) + { + return container.Elements().Where(e => e.Name.LocalName == elementName.LocalName); + } + + public static XElement ElementIgnoringNamespace(this XContainer container, XName elementName) + { + return container.ElementsIgnroingNamespace(elementName).FirstOrDefault(); + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/GetSourceBuiltNupkgCacheConflicts.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/GetSourceBuiltNupkgCacheConflicts.cs new file mode 100644 index 000000000..7eb24ae1a --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/GetSourceBuiltNupkgCacheConflicts.cs @@ -0,0 +1,153 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Packaging.Core; +using NuGet.Versioning; +using System; +using System.IO; +using System.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks +{ + /// + /// For each source-built nupkg info given, ensure that if the package cache contains a package + /// with the same id and version, the cached nupkg is the same as the source-built one. + /// + /// If the package cache contains a package with the same package id and version as a + /// source-built one, nuget restore short-circuits and doesn't look for the source-built one. + /// This usually results in prebuilt packages being used, which can either break the build or + /// end up in the outputs. + /// + public class GetSourceBuiltNupkgCacheConflicts : Task + { + /// + /// Items containing package id and version of each source-built package. + /// ReadNuGetPackageInfos is recommended to generate these. + /// + /// %(Identity): Path to the original nupkg. + /// %(PackageId): Identity of the package. + /// %(PackageVersion): Version of the package. + /// + [Required] + public ITaskItem[] SourceBuiltPackageInfos { get; set; } + + /// + /// Package cache dir containing nupkgs to compare. Path is expected to be like: + /// + /// {PackageCacheDir}/{lowercase id}/{version}/{lowercase id}.{version}.nupkg + /// + [Required] + public string PackageCacheDir { get; set; } + + /// + /// Paths to packages to compare against when conflicts are detected. Knowing where the + /// package in the cache came from can help diagnose a conflict. For example, is it from + /// prebuilt/source-built? Or does the build not have the nupkg anywhere else, and + /// therefore it most likely came from the internet? + /// + public string[] KnownOriginPackagePaths { get; set; } + + [Output] + public ITaskItem[] ConflictingPackageInfos { get; set; } + + public override bool Execute() + { + DateTime startTime = DateTime.Now; + + var knownNupkgs = new Lazy>(() => + { + Log.LogMessage( + MessageImportance.Low, + $"Reading all {nameof(KnownOriginPackagePaths)} package identities to search " + + "for conflicting package origin..."); + + return KnownOriginPackagePaths.NullAsEmpty().ToLookup( + ReadNuGetPackageInfos.ReadIdentity, + path => path); + }); + + ConflictingPackageInfos = SourceBuiltPackageInfos + .Where(item => + { + string sourceBuiltPath = item.ItemSpec; + string id = item.GetMetadata("PackageId"); + string version = item.GetMetadata("PackageVersion"); + + string packageCachePath = Path.Combine( + PackageCacheDir, + id.ToLowerInvariant(), + version, + $"{id.ToLowerInvariant()}.{version}.nupkg"); + + if (!File.Exists(packageCachePath)) + { + Log.LogMessage( + MessageImportance.Low, + $"OK: Package not found in package cache: {id} {version}"); + return false; + } + + Log.LogMessage( + MessageImportance.Low, + $"Package id/version found in package cache, verifying: {id} {version}"); + + byte[] packageCacheBytes = File.ReadAllBytes(packageCachePath); + + if (packageCacheBytes.SequenceEqual(File.ReadAllBytes(sourceBuiltPath))) + { + Log.LogMessage( + MessageImportance.Low, + $"OK: Package in cache is identical to source-built: {id} {version}"); + return false; + } + + Log.LogMessage( + MessageImportance.Low, + "BAD: Source-built nupkg is not byte-for-byte identical " + + $"to nupkg in cache: {id} {version}"); + + var ident = new PackageIdentity(id, NuGetVersion.Parse(version)); + + string message = null; + + foreach (string knownNupkg in knownNupkgs.Value[ident]) + { + if (packageCacheBytes.SequenceEqual(File.ReadAllBytes(knownNupkg))) + { + Log.LogMessage( + MessageImportance.Low, + $"Found identity match with identical contents: {knownNupkg}"); + + message = (message ?? "Nupkg found at") + $" '{knownNupkg}'"; + } + else + { + Log.LogMessage( + MessageImportance.Low, + $"Package identity match, but contents differ: {knownNupkg}"); + } + } + + item.SetMetadata( + "WarningMessage", + message ?? + "Origin nupkg not found in build directory. It may have been " + + "downloaded by NuGet restore."); + + return true; + }) + .ToArray(); + + // Tell the user about this task, in case it takes a while. + Log.LogMessage( + MessageImportance.High, + "Checked cache for conflicts with source-built nupkgs. " + + $"Took {DateTime.Now - startTime}"); + + return !Log.HasLoggedErrors; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Log.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Log.cs new file mode 100644 index 000000000..83d587c0a --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Log.cs @@ -0,0 +1,118 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks.Packaging +{ + internal class Log : ILog + { + private readonly TaskLoggingHelper _logger; + public Log(TaskLoggingHelper logger) + { + _logger = logger; + } + + public void LogError(string message, params object[] messageArgs) + { + _logger.LogError(message, messageArgs); + } + + public void LogMessage(string message, params object[] messageArgs) + { + _logger.LogMessage(message, messageArgs); + } + + public void LogMessage(LogImportance importance, string message, params object[] messageArgs) + { + _logger.LogMessage((MessageImportance)importance, message, messageArgs); + } + + public void LogWarning(string message, params object[] messageArgs) + { + _logger.LogWarning(message, messageArgs); + } + + public bool HasLoggedErrors { get { return _logger.HasLoggedErrors; } } + } + + public enum LogImportance + { + Low = MessageImportance.Low, + Normal = MessageImportance.Normal, + High = MessageImportance.High + } + + + public interface ILog + { + // + // Summary: + // Logs an error with the specified message. + // + // Parameters: + // message: + // The message. + // + // messageArgs: + // Optional arguments for formatting the message string. + // + // Exceptions: + // T:System.ArgumentNullException: + // message is null. + void LogError(string message, params object[] messageArgs); + + // + // Summary: + // Logs a message with the specified string. + // + // Parameters: + // message: + // The message. + // + // messageArgs: + // The arguments for formatting the message. + // + // Exceptions: + // T:System.ArgumentNullException: + // message is null. + void LogMessage(string message, params object[] messageArgs); + + // + // Summary: + // Logs a message with the specified string and importance. + // + // Parameters: + // importance: + // One of the enumeration values that specifies the importance of the message. + // + // message: + // The message. + // + // messageArgs: + // The arguments for formatting the message. + // + // Exceptions: + // T:System.ArgumentNullException: + // message is null. + void LogMessage(LogImportance importance, string message, params object[] messageArgs); + + // + // Summary: + // Logs a warning with the specified message. + // + // Parameters: + // message: + // The message. + // + // messageArgs: + // Optional arguments for formatting the message string. + // + // Exceptions: + // T:System.ArgumentNullException: + // message is null. + void LogWarning(string message, params object[] messageArgs); + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Microsoft.DotNet.SourceBuild.Tasks.XPlat.csproj b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Microsoft.DotNet.SourceBuild.Tasks.XPlat.csproj new file mode 100755 index 000000000..08d19f382 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Microsoft.DotNet.SourceBuild.Tasks.XPlat.csproj @@ -0,0 +1,27 @@ + + + + netcoreapp5.0 + $(XPlatTasksBinDir) + + + + + 15.7.179 + + + 15.7.179 + + + 15.7.179 + + + 15.7.179 + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Models/VersionDetailsXml.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Models/VersionDetailsXml.cs new file mode 100644 index 000000000..89e78085e --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/Models/VersionDetailsXml.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Xml.Serialization; + +namespace Microsoft.DotNet.SourceBuild.Tasks.Models +{ + [XmlRoot("Dependencies")] + public class VersionDetails + { + [XmlArray("ToolsetDependencies")] + public Dependency[] ToolsetDependencies { get; set; } + [XmlArray("ProductDependencies")] + public Dependency[] ProductDependencies { get; set; } + } + + public class Dependency + { + [XmlAttribute] + public string Name { get; set; } + [XmlAttribute] + public string Version { get; set; } + [XmlAttribute] + public string CoherentParentDependency { get; set; } + [XmlAttribute] + public bool Pinned { get; set; } + // Uri type isn't serializable, so use a string instead + public string Uri { get; set; } + public string Sha { get; set; } + [XmlElement("RepoName")] + public string[] RepoNames { get; set; } + + public override string ToString() + { + return $"{Name}@{Version} ({Uri}@{Sha})"; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/NuGetPack.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/NuGetPack.cs new file mode 100644 index 000000000..d463f53e8 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/NuGetPack.cs @@ -0,0 +1,376 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using NuGet; +using NuGet.Versioning; +using NuGet.Packaging; +using System; +using System.IO; +using System.Collections.Generic; +using System.Linq; +using NuGet.Common; + +namespace Microsoft.DotNet.Build.Tasks.Packaging +{ + public class NuGetPack : PackagingTask + { + /// + /// Target file paths to exclude when building the lib package for symbol server scenario + /// Copied from https://github.com/NuGet/NuGet.Client/blob/59433c7bacaae435a2cfe343cd441ea710579304/src/NuGet.Core/NuGet.Commands/PackCommandRunner.cs#L48 + /// + private static readonly string[] _libPackageExcludes = new[] { + @"**\*.pdb".Replace('\\', Path.DirectorySeparatorChar), + @"src\**\*".Replace('\\', Path.DirectorySeparatorChar) + }; + + /// + /// Target file paths to exclude when building the symbols package for symbol server scenario + /// + private static readonly string[] _symbolPackageExcludes = new[] { + @"content\**\*".Replace('\\', Path.DirectorySeparatorChar), + @"tools\**\*.ps1".Replace('\\', Path.DirectorySeparatorChar) + }; + + private static readonly string _defaultPackedPackagePrefix = "transport"; + private static readonly string _symbolsPackageExtension = ".symbols.nupkg"; + private static readonly string _packageExtension = ".nupkg"; + + [Required] + public ITaskItem[] Nuspecs + { + get; + set; + } + + [Required] + public string OutputDirectory + { + get; + set; + } + + public string BaseDirectory + { + get; + set; + } + + public string PackageVersion + { + get; + set; + } + + public bool ExcludeEmptyDirectories + { + get; + set; + } + // Create an additional ".symbols.nupkg" package + public bool CreateSymbolPackage + { + get; + set; + } + // Include symbols in standard package + public bool IncludeSymbolsInPackage + { + get; + set; + } + // Create an additional "packed package" that includes lib and src / symbols + public bool CreatePackedPackage + { + get; + set; + } + /// + /// Nuspec files can contain properties that are substituted with values at pack time + /// This task property passes through the nuspect properties. + /// Each item is a string with the syntax = + /// String validation for and is deffered to the Nuget APIs + /// + public ITaskItem[] NuspecProperties + { + get; + set; + } + + public ITaskItem[] AdditionalLibPackageExcludes + { + get; + set; + } + + public ITaskItem[] AdditionalSymbolPackageExcludes + { + get; + set; + } + + /// + /// If set, the symbol package is placed in the given directory. Otherwise OutputDirectory is used. + /// + public string SymbolPackageOutputDirectory + { + get; + set; + } + + public string PackedPackageNamePrefix + { + get; + set; + } + + public override bool Execute() + { + if (Nuspecs == null || Nuspecs.Length == 0) + { + Log.LogError("Nuspecs argument must be specified"); + return false; + } + + if (String.IsNullOrEmpty(OutputDirectory)) + { + Log.LogError("OuputDirectory argument must be specified"); + return false; + } + + if (!Directory.Exists(OutputDirectory)) + { + Directory.CreateDirectory(OutputDirectory); + } + + Func nuspecPropertyProvider = GetNuspecPropertyProviderFunction(NuspecProperties); + + foreach (var nuspec in Nuspecs) + { + string nuspecPath = nuspec.GetMetadata("FullPath"); + + if (!File.Exists(nuspecPath)) + { + Log.LogError($"Nuspec {nuspecPath} does not exist"); + continue; + } + + Manifest manifest = GetManifest(nuspecPath, nuspecPropertyProvider, false); + string nupkgPath = GetPackageOutputPath(nuspecPath, manifest, false, false); + Pack(nuspecPath, nupkgPath, manifest, IncludeSymbolsInPackage); + + bool packSymbols = CreateSymbolPackage || CreatePackedPackage; + if (CreateSymbolPackage) + { + Manifest symbolsManifest = GetManifest(nuspecPath, nuspecPropertyProvider, false); + nupkgPath = GetPackageOutputPath(nuspecPath, symbolsManifest, true, false); + Pack(nuspecPath, nupkgPath, symbolsManifest, packSymbols); + } + + if (CreatePackedPackage) + { + Manifest packedManifest = GetManifest(nuspecPath, nuspecPropertyProvider, true); + nupkgPath = GetPackageOutputPath(nuspecPath, packedManifest, false, true); + Pack(nuspecPath, nupkgPath, packedManifest, packSymbols); + } + } + + return !Log.HasLoggedErrors; + } + + private static Func GetNuspecPropertyProviderFunction(ITaskItem[] nuspecProperties) + { + return nuspecProperties == null ? null : NuspecPropertyStringProvider.GetNuspecPropertyProviderFunction(nuspecProperties.Select(p => p.ItemSpec).ToArray()); + } + + private Manifest GetManifest(string nuspecPath, Func nuspecPropertyProvider, bool isPackedPackage) + { + using (var nuspecFile = File.Open(nuspecPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete)) + { + string baseDirectoryPath = (string.IsNullOrEmpty(BaseDirectory)) ? Path.GetDirectoryName(nuspecPath) : BaseDirectory; + Manifest manifest = Manifest.ReadFrom(nuspecFile, nuspecPropertyProvider, false); + + if (isPackedPackage) + { + manifest = TransformManifestToPackedPackageManifest(manifest); + } + return manifest; + } + } + + private string GetPackageOutputPath(string nuspecPath, Manifest manifest, bool isSymbolsPackage, bool applyPrefix) + { + string id = manifest.Metadata.Id; + + if (String.IsNullOrEmpty(id)) + { + Log.LogError($"Nuspec {nuspecPath} does not contain a valid Id"); + return string.Empty; + } + + // Overriding the Version from the Metadata if one gets passed in. + if (!string.IsNullOrEmpty(PackageVersion)) + { + NuGetVersion overrideVersion; + if (NuGetVersion.TryParse(PackageVersion, out overrideVersion)) + { + manifest.Metadata.Version = overrideVersion; + } + else + { + Log.LogError($"Failed to parse Package Version: '{PackageVersion}' is not a valid version."); + } + } + + string version = manifest.Metadata.Version.ToString(); + + if (String.IsNullOrEmpty(version)) + { + Log.LogError($"Nuspec {nuspecPath} does not contain a valid version"); + return string.Empty; + } + + string nupkgOutputDirectory = OutputDirectory; + + if (isSymbolsPackage && !string.IsNullOrEmpty(SymbolPackageOutputDirectory)) + { + nupkgOutputDirectory = SymbolPackageOutputDirectory; + } + + string nupkgExtension = isSymbolsPackage ? _symbolsPackageExtension : _packageExtension; + return Path.Combine(nupkgOutputDirectory, $"{id}.{version}{nupkgExtension}"); + } + + public void Pack(string nuspecPath, string nupkgPath, Manifest manifest, bool packSymbols) + { + bool creatingSymbolsPackage = packSymbols && (Path.GetExtension(nupkgPath) == _symbolsPackageExtension); + try + { + PackageBuilder builder = new PackageBuilder(); + + string baseDirectoryPath = (string.IsNullOrEmpty(BaseDirectory)) ? Path.GetDirectoryName(nuspecPath) : BaseDirectory; + builder.Populate(manifest.Metadata); + builder.PopulateFiles(baseDirectoryPath, manifest.Files); + + if (creatingSymbolsPackage) + { + // For symbols packages, filter out excludes + PathResolver.FilterPackageFiles( + builder.Files, + file => file.Path, + SymbolPackageExcludes); + + // Symbol packages are only valid if they contain both symbols and sources. + Dictionary pathHasMatches = LibPackageExcludes.ToDictionary( + path => path, + path => PathResolver.GetMatches(builder.Files, file => file.Path, new[] { path }).Any()); + + if (!pathHasMatches.Values.Any(i => i)) + { + Log.LogMessage(LogImportance.Low, $"Nuspec {nuspecPath} does not contain symbol or source files. Not creating symbol package."); + return; + } + foreach (var pathPair in pathHasMatches.Where(pathMatchPair => !pathMatchPair.Value)) + { + Log.LogMessage(LogImportance.Low, $"Nuspec {nuspecPath} does not contain any files matching {pathPair.Key}. Not creating symbol package."); + return; + } + } + else if(!packSymbols) + { + // for packages which do not include symbols (not symbols or packed packages), filter lib excludes + PathResolver.FilterPackageFiles( + builder.Files, + file => file.Path, + LibPackageExcludes); + } + + var directory = Path.GetDirectoryName(nupkgPath); + if (!Directory.Exists(directory)) + { + Directory.CreateDirectory(directory); + } + + using (var fileStream = File.Create(nupkgPath)) + { + builder.Save(fileStream); + } + + Log.LogMessage($"Created '{nupkgPath}'"); + } + catch (Exception e) + { + string packageType = "lib"; + if (creatingSymbolsPackage) + { + packageType = "symbol"; + } + else if (packSymbols) + { + packageType = "packed"; + } + Log.LogError($"Error when creating nuget {packageType} package from {nuspecPath}. {e}"); + } + } + + private Manifest TransformManifestToPackedPackageManifest(Manifest manifest) + { + ManifestMetadata manifestMetadata = manifest.Metadata; + + // Update Id + string _packageNamePrefix = PackedPackageNamePrefix != null ? PackedPackageNamePrefix : _defaultPackedPackagePrefix; + manifestMetadata.Id = $"{_packageNamePrefix}.{manifestMetadata.Id}"; + + // Update dependencies + List packedPackageDependencyGroups = new List(); + foreach(var dependencyGroup in manifestMetadata.DependencyGroups) + { + List packages = new List(); + foreach(var dependency in dependencyGroup.Packages) + { + NuGet.Packaging.Core.PackageDependency package = new NuGet.Packaging.Core.PackageDependency($"{_packageNamePrefix}.{dependency.Id}", dependency.VersionRange, dependency.Include, dependency.Exclude); + packages.Add(package); + } + PackageDependencyGroup packageDependencyGroup = new PackageDependencyGroup(dependencyGroup.TargetFramework, packages); + packedPackageDependencyGroups.Add(packageDependencyGroup); + } + manifestMetadata.DependencyGroups = packedPackageDependencyGroups; + + // Update runtime.json + List manifestFiles = new List(); + + foreach(ManifestFile file in manifest.Files) + { + string fileName = file.Source; + if(Path.GetFileName(fileName) == "runtime.json" && file.Target == "") + { + string packedPackageSourcePath = Path.Combine(Path.GetDirectoryName(fileName), string.Join(".", _packageNamePrefix, Path.GetFileName(fileName))); + file.Source = File.Exists(packedPackageSourcePath) ? packedPackageSourcePath : fileName; + file.Target = "runtime.json"; + } + manifestFiles.Add(file); + } + Manifest packedPackageManifest = new Manifest(manifestMetadata, manifestFiles); + return manifest; + } + + private IEnumerable LibPackageExcludes + { + get + { + return _libPackageExcludes + .Concat(AdditionalLibPackageExcludes?.Select(item => item.ItemSpec) ?? Enumerable.Empty()); + } + } + + private IEnumerable SymbolPackageExcludes + { + get + { + return _symbolPackageExcludes + .Concat(AdditionalSymbolPackageExcludes?.Select(item => item.ItemSpec) ?? Enumerable.Empty()); + } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/NuspecPropertyStringProvider.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/NuspecPropertyStringProvider.cs new file mode 100644 index 000000000..d10fad1b2 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/NuspecPropertyStringProvider.cs @@ -0,0 +1,72 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Collections.Generic; + +namespace Microsoft.DotNet.Build.Tasks.Packaging +{ + public class NuspecPropertyStringProvider + { + public static Dictionary GetNuspecPropertyDictionary(string[] nuspecProperties) + { + if (nuspecProperties == null) + { + return null; + } + + var propertyDictionary = new Dictionary(); + foreach (var propertyString in nuspecProperties) + { + var property = GetKeyValuePair(propertyString); + propertyDictionary[property.Item1] = property.Item2; + } + + return propertyDictionary; + } + + public static Func GetNuspecPropertyProviderFunction(string[] nuspecPropertyStrings) + { + var propertyDictionary = GetNuspecPropertyDictionary(nuspecPropertyStrings); + + if (propertyDictionary == null) + { + return null; + } + + return k => propertyDictionary[k]; + } + + private static Tuple GetKeyValuePair(string propertyString) + { + propertyString = propertyString.Trim(); + + var indexOfEquals = propertyString.IndexOf("=", StringComparison.Ordinal); + + if (indexOfEquals == -1) + { + throw new InvalidDataException($"Nuspec property {propertyString} does not have an \'=\' character in it"); + } + + if (indexOfEquals == propertyString.Length - 1) + { + throw new InvalidDataException($"Nuspec property {propertyString} does not have a value"); + } + + if (indexOfEquals == 0) + { + throw new InvalidDataException($"Nuspec property {propertyString} does not have a key"); + } + + var key = propertyString.Substring(0, indexOfEquals); + + var valueStartIndex = indexOfEquals + 1; + var valueLength = propertyString.Length - valueStartIndex; + var value = propertyString.Substring(valueStartIndex, valueLength); + + return new Tuple(key, value); + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/PackagingTask.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/PackagingTask.cs new file mode 100644 index 000000000..24e2e0047 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/PackagingTask.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks.Packaging +{ + public abstract partial class PackagingTask : ITask + { + private Log _log = null; + + internal Log Log + { + get { return _log ?? (_log = new Log(new TaskLoggingHelper(this))); } + } + + public PackagingTask() + { + } + + public IBuildEngine BuildEngine + { + get; + set; + } + + public ITaskHost HostObject + { + get; + set; + } + + public abstract bool Execute(); + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReadNuGetPackageInfos.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReadNuGetPackageInfos.cs new file mode 100644 index 000000000..43a04ebd0 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReadNuGetPackageInfos.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks +{ + public class ReadNuGetPackageInfos : Task + { + [Required] + public string[] PackagePaths { get; set; } + + /// + /// %(Identity): Path to the original nupkg. + /// %(PackageId): Identity of the package. + /// %(PackageVersion): Version of the package. + /// + [Output] + public ITaskItem[] PackageInfoItems { get; set; } + + public override bool Execute() + { + PackageInfoItems = PackagePaths + .Select(p => + { + PackageIdentity identity = ReadIdentity(p); + return new TaskItem( + p, + new Dictionary + { + ["PackageId"] = identity.Id, + ["PackageVersion"] = identity.Version.OriginalVersion + }); + }) + .ToArray(); + + return !Log.HasLoggedErrors; + } + + public static PackageIdentity ReadIdentity(string nupkgFile) + { + using (var reader = new PackageArchiveReader(nupkgFile)) + { + return reader.GetIdentity(); + } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/RemoveInternetSourcesFromNuGetConfig.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/RemoveInternetSourcesFromNuGetConfig.cs new file mode 100755 index 000000000..65aa69ac8 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/RemoveInternetSourcesFromNuGetConfig.cs @@ -0,0 +1,83 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + /* + * This task removes internet sources from a given NuGet.config. In the offline build mode, it removes all + * feeds that begin with http or https. In the online build mode, it removes only the internal dnceng feeds that + * source-build does not have access to. + */ + public class RemoveInternetSourcesFromNuGetConfig : Task + { + [Required] + public string NuGetConfigFile { get; set; } + + /// + /// Whether to work in offline mode (remove all internet sources) or online mode (remove only authenticated sources) + /// + public bool OfflineBuild { get; set; } + + /// + /// A list of prefix strings that make the task keep a package source unconditionally. For + /// example, a source named 'darc-pub-dotnet-aspnetcore-e81033e' will be kept if the prefix + /// 'darc-pub-dotnet-aspnetcore-' is in this list. + /// + public string[] KeepFeedPrefixes { get; set; } + + public override bool Execute() + { + XDocument d = XDocument.Load(NuGetConfigFile); + XElement packageSourcesElement = d.Root.Descendants().First(e => e.Name == "packageSources"); + XElement disabledPackageSourcesElement = d.Root.Descendants().FirstOrDefault(e => e.Name == "disabledPackageSources"); + + IEnumerable local = packageSourcesElement.Descendants().Where(e => + { + if (e.Name == "add") + { + string feedName = e.Attribute("key").Value; + if (KeepFeedPrefixes + ?.Any(prefix => feedName.StartsWith(prefix, StringComparison.OrdinalIgnoreCase)) + == true) + { + return true; + } + + string feedUrl = e.Attribute("value").Value; + if (OfflineBuild) + { + return !(feedUrl.StartsWith("http://", StringComparison.OrdinalIgnoreCase) || feedUrl.StartsWith("https://", StringComparison.OrdinalIgnoreCase)); + } + else + { + return !( feedUrl.StartsWith("https://pkgs.dev.azure.com/dnceng/_packaging", StringComparison.OrdinalIgnoreCase) || + feedUrl.StartsWith("https://pkgs.dev.azure.com/dnceng/internal/_packaging", StringComparison.OrdinalIgnoreCase) ); + } + } + + return true; + }); + + packageSourcesElement.ReplaceNodes(local.ToArray()); + + // Remove disabledPackageSources element so if any internal packages remain, they are used in source-build + disabledPackageSourcesElement?.ReplaceNodes(new XElement("clear")); + + using (FileStream fs = new FileStream(NuGetConfigFile, FileMode.Create, FileAccess.ReadWrite)) + { + d.Save(fs); + } + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceFeedsInNuGetConfig.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceFeedsInNuGetConfig.cs new file mode 100644 index 000000000..a4519ead5 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceFeedsInNuGetConfig.cs @@ -0,0 +1,54 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + /// + /// Replaces feeds in a NuGet.Config file given a mapping + /// of old feeds to new feeds. + /// + public class ReplaceFeedsInNuGetConfig : Task + { + /// + /// The NuGet.Config file in which to replace feeds. + /// + [Required] + public string InputFile { get; set; } + + /// + /// An item group of feeds to update. + /// %(Identity): The feed URL to find in the NuGet.Config. + /// %(NewFeed): The feed URL to replace %(Identity) with. + /// + [Required] + public ITaskItem[] FeedMapping { get; set; } + + public override bool Execute() + { + string fileContents = File.ReadAllText(InputFile); + bool updated = false; + + foreach (var feed in FeedMapping) + { + string oldFeed = feed.ItemSpec; + string newFeed = feed.GetMetadata("NewFeed"); + + if (fileContents.Contains(oldFeed)) + { + fileContents = fileContents.Replace(oldFeed, newFeed); + updated = true; + } + } + + if (updated) File.WriteAllText(InputFile, fileContents); + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceRegexInFiles.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceRegexInFiles.cs new file mode 100644 index 000000000..9b0d2e10c --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceRegexInFiles.cs @@ -0,0 +1,39 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Text.RegularExpressions; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class ReplaceRegexInFiles : Task + { + [Required] + public string[] InputFiles { get; set; } + + [Required] + public string OldTextRegex { get; set; } + + [Required] + public string NewText { get; set; } + + public override bool Execute() + { + Log.LogMessage($"Replacing '{OldTextRegex}' with '{NewText}'"); + foreach (string file in InputFiles) + { + string fileContents = File.ReadAllText(file); + + fileContents = Regex.Replace(fileContents, OldTextRegex, NewText); + + File.WriteAllText(file, fileContents); + } + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceTextInFile.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceTextInFile.cs new file mode 100644 index 000000000..32b8cdea1 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceTextInFile.cs @@ -0,0 +1,35 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class ReplaceTextInFile : Task + { + [Required] + public string InputFile { get; set; } + + [Required] + public string OldText { get; set; } + + [Required] + public string NewText { get; set; } + + + public override bool Execute() + { + string fileContents = File.ReadAllText(InputFile); + + fileContents = fileContents.Replace(OldText, NewText); + + File.WriteAllText(InputFile, fileContents); + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceTextInFiles.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceTextInFiles.cs new file mode 100644 index 000000000..30f1d6e67 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ReplaceTextInFiles.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class ReplaceTextInFiles : Task + { + [Required] + public string[] InputFiles { get; set; } + + [Required] + public string OldText { get; set; } + + [Required] + public string NewText { get; set; } + + public override bool Execute() + { + foreach (string file in InputFiles) + { + string fileContents = File.ReadAllText(file); + + fileContents = fileContents.Replace(OldText, NewText, StringComparison.Ordinal); + + File.WriteAllText(file, fileContents); + } + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/RepoTasks/JoinItems.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/RepoTasks/JoinItems.cs new file mode 100644 index 000000000..55f941c16 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/RepoTasks/JoinItems.cs @@ -0,0 +1,139 @@ +// Copyright (c) .NET Foundation. All rights reserved. +// Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information. + +// Copied from https://github.com/aspnet/Universe/blob/1f8f30a1e834eff147ced0c669cef8828f9511c8/build/tasks/JoinItems.cs. +// When this task is available in https://github.com/dotnet/Arcade, switch to use that version. +// Modified to allow multiple Right matches using GroupJoin. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System; +using System.Linq; + +namespace RepoTasks +{ + public class JoinItems : Task + { + [Required] + public ITaskItem[] Left { get; set; } + + [Required] + public ITaskItem[] Right { get; set; } + + // The metadata to use as the new item spec. If not specified, LeftKey is used. + public string LeftItemSpec { get; set; } + + // LeftKey and RightKey: The metadata to join on. If not set, then use the ItemSpec + public string LeftKey { get; set; } + + public string RightKey { get; set; } + + + // LeftMetadata and RightMetadata: The metadata names to include in the result. Specify "*" to include all metadata + public string[] LeftMetadata { get; set; } + + public string[] RightMetadata { get; set; } + + + [Output] + public ITaskItem[] JoinResult { get; private set; } + + public override bool Execute() + { + bool useAllLeftMetadata = LeftMetadata != null && LeftMetadata.Length == 1 && LeftMetadata[0] == "*"; + bool useAllRightMetadata = RightMetadata != null && RightMetadata.Length == 1 && RightMetadata[0] == "*"; + var newItemSpec = string.IsNullOrEmpty(LeftItemSpec) + ? LeftKey + : LeftItemSpec; + + JoinResult = Left.GroupJoin(Right, + item => GetKeyValue(LeftKey, item), + item => GetKeyValue(RightKey, item), + (left, rights) => + { + // If including all metadata from left items and none from right items, just return left items directly + if (useAllLeftMetadata && + string.IsNullOrEmpty(LeftKey) && + string.IsNullOrEmpty(LeftItemSpec) && + (RightMetadata == null || RightMetadata.Length == 0)) + { + return left; + } + + // If including all metadata from all right items and none from left items, just return the right items directly + if (useAllRightMetadata && + string.IsNullOrEmpty(RightKey) && + string.IsNullOrEmpty(LeftItemSpec) && + (LeftMetadata == null || LeftMetadata.Length == 0)) + { + return rights.Aggregate( + new TaskItem(), + (agg, next) => + { + CopyAllMetadata(next, agg); + return agg; + }); + } + + var ret = new TaskItem(GetKeyValue(newItemSpec, left)); + + // Weird ordering here is to prefer left metadata in all cases, as CopyToMetadata doesn't overwrite any existing metadata + if (useAllLeftMetadata) + { + CopyAllMetadata(left, ret); + } + + if (!useAllRightMetadata && RightMetadata != null) + { + foreach (string name in RightMetadata) + { + foreach (var right in rights) + { + ret.SetMetadata(name, right.GetMetadata(name)); + } + } + } + + if (!useAllLeftMetadata && LeftMetadata != null) + { + foreach (string name in LeftMetadata) + { + ret.SetMetadata(name, left.GetMetadata(name)); + } + } + + if (useAllRightMetadata) + { + foreach (var right in rights) + { + CopyAllMetadata(right, ret); + } + } + + return (ITaskItem)ret; + }, + StringComparer.OrdinalIgnoreCase).ToArray(); + + return true; + } + + static void CopyAllMetadata(ITaskItem source, ITaskItem dest) + { + // CopyMetadata adds an OriginalItemSpec, which we don't want. So we subsequently remove it + source.CopyMetadataTo(dest); + dest.RemoveMetadata("OriginalItemSpec"); + } + + static string GetKeyValue(string key, ITaskItem item) + { + if (string.IsNullOrEmpty(key)) + { + return item.ItemSpec; + } + else + { + return item.GetMetadata(key); + } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateJson.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateJson.cs new file mode 100644 index 000000000..9820e4af2 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UpdateJson.cs @@ -0,0 +1,69 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Linq; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; + +namespace Microsoft.DotNet.Build.Tasks +{ + // Takes a path to a path to a json file and a + // string that represents a dotted path to an attribute + // and updates that attribute with the new value provided. + public class UpdateJson : Task + { + [Required] + public string JsonFilePath { get; set; } + + [Required] + public string PathToAttribute { get; set; } + + [Required] + public string NewAttributeValue { get; set; } + + public bool SkipUpdateIfMissingKey { get; set; } + + public override bool Execute() + { + JObject jsonObj = JObject.Parse(File.ReadAllText(JsonFilePath)); + + string[] escapedPathToAttributeParts = PathToAttribute.Replace("\\.", "\x1F").Split('.'); + for (int i = 0; i < escapedPathToAttributeParts.Length; ++i) + { + escapedPathToAttributeParts[i] = escapedPathToAttributeParts[i].Replace("\x1F", "."); + } + UpdateAttribute(jsonObj, escapedPathToAttributeParts, NewAttributeValue); + + File.WriteAllText(JsonFilePath, jsonObj.ToString()); + return true; + } + + private void UpdateAttribute(JToken jsonObj, string[] path, string newValue) + { + string pathItem = path[0]; + if (jsonObj[pathItem] == null) + { + string message = $"Path item [{nameof(PathToAttribute)}] not found in json file."; + if (SkipUpdateIfMissingKey) + { + Log.LogMessage(MessageImportance.Low, $"Skipping update: {message} {pathItem}"); + return; + } + throw new ArgumentException(message, pathItem); + } + + if (path.Length == 1) + { + jsonObj[pathItem] = newValue; + return; + } + + UpdateAttribute(jsonObj[pathItem], path.Skip(1).ToArray(), newValue); + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadClient.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadClient.cs new file mode 100644 index 000000000..c02055705 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadClient.cs @@ -0,0 +1,285 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using Task = System.Threading.Tasks.Task; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class UploadClient + { + private TaskLoggingHelper log; + + public UploadClient(TaskLoggingHelper loggingHelper) + { + log = loggingHelper; + } + + public string EncodeBlockIds(int numberOfBlocks, int lengthOfId) + { + string numberOfBlocksString = numberOfBlocks.ToString("D" + lengthOfId); + if (Encoding.UTF8.GetByteCount(numberOfBlocksString) <= 64) + { + byte[] bytes = Encoding.UTF8.GetBytes(numberOfBlocksString); + return Convert.ToBase64String(bytes); + } + else + { + throw new Exception("Task failed - Could not encode block id."); + } + } + + public async Task UploadBlockBlobAsync( + CancellationToken ct, + string AccountName, + string AccountKey, + string ContainerName, + string filePath, + string destinationBlob, + string contentType, + int uploadTimeout, + string leaseId = "") + { + string resourceUrl = AzureHelper.GetContainerRestUrl(AccountName, ContainerName); + + string fileName = destinationBlob; + fileName = fileName.Replace("\\", "/"); + string blobUploadUrl = resourceUrl + "/" + fileName; + int size = (int)new FileInfo(filePath).Length; + int blockSize = 4 * 1024 * 1024; //4MB max size of a block blob + int bytesLeft = size; + List blockIds = new List(); + int numberOfBlocks = (size / blockSize) + 1; + int countForId = 0; + using (FileStream fileStreamTofilePath = new FileStream(filePath, FileMode.Open, FileAccess.Read)) + { + int offset = 0; + + while (bytesLeft > 0) + { + int nextBytesToRead = (bytesLeft < blockSize) ? bytesLeft : blockSize; + byte[] fileBytes = new byte[blockSize]; + int read = fileStreamTofilePath.Read(fileBytes, 0, nextBytesToRead); + + if (nextBytesToRead != read) + { + throw new Exception(string.Format( + "Number of bytes read ({0}) from file {1} isn't equal to the number of bytes expected ({2}) .", + read, fileName, nextBytesToRead)); + } + + string blockId = EncodeBlockIds(countForId, numberOfBlocks.ToString().Length); + + blockIds.Add(blockId); + string blockUploadUrl = blobUploadUrl + "?comp=block&blockid=" + WebUtility.UrlEncode(blockId); + + using (HttpClient client = new HttpClient()) + { + client.DefaultRequestHeaders.Clear(); + + // In random occassions the request fails if the network is slow and it takes more than 100 seconds to upload 4MB. + client.Timeout = TimeSpan.FromMinutes(uploadTimeout); + Func createRequest = () => + { + DateTime dt = DateTime.UtcNow; + var req = new HttpRequestMessage(HttpMethod.Put, blockUploadUrl); + req.Headers.Add( + AzureHelper.DateHeaderString, + dt.ToString("R", CultureInfo.InvariantCulture)); + req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); + if (!string.IsNullOrWhiteSpace(leaseId)) + { + log.LogMessage($"Sending request: {leaseId} {blockUploadUrl}"); + req.Headers.Add("x-ms-lease-id", leaseId); + } + req.Headers.Add( + AzureHelper.AuthorizationHeaderString, + AzureHelper.AuthorizationHeader( + AccountName, + AccountKey, + "PUT", + dt, + req, + string.Empty, + string.Empty, + nextBytesToRead.ToString(), + string.Empty)); + + Stream postStream = new MemoryStream(); + postStream.Write(fileBytes, 0, nextBytesToRead); + postStream.Seek(0, SeekOrigin.Begin); + req.Content = new StreamContent(postStream); + return req; + }; + + log.LogMessage(MessageImportance.Low, "Sending request to upload part {0} of file {1}", countForId, fileName); + + using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest)) + { + log.LogMessage( + MessageImportance.Low, + "Received response to upload part {0} of file {1}: Status Code:{2} Status Desc: {3}", + countForId, + fileName, + response.StatusCode, + await response.Content.ReadAsStringAsync()); + } + } + + offset += read; + bytesLeft -= nextBytesToRead; + countForId += 1; + } + } + + string blockListUploadUrl = blobUploadUrl + "?comp=blocklist"; + + using (HttpClient client = new HttpClient()) + { + Func createRequest = () => + { + DateTime dt1 = DateTime.UtcNow; + var req = new HttpRequestMessage(HttpMethod.Put, blockListUploadUrl); + req.Headers.Add(AzureHelper.DateHeaderString, dt1.ToString("R", CultureInfo.InvariantCulture)); + req.Headers.Add(AzureHelper.VersionHeaderString, AzureHelper.StorageApiVersion); + if (string.IsNullOrEmpty(contentType)) + { + contentType = DetermineContentTypeBasedOnFileExtension(filePath); + } + if (!string.IsNullOrEmpty(contentType)) + { + req.Headers.Add(AzureHelper.ContentTypeString, contentType); + } + string cacheControl = DetermineCacheControlBasedOnFileExtension(filePath); + if (!string.IsNullOrEmpty(cacheControl)) + { + req.Headers.Add(AzureHelper.CacheControlString, cacheControl); + } + + var body = new StringBuilder(""); + foreach (object item in blockIds) + body.AppendFormat("{0}", item); + + body.Append(""); + byte[] bodyData = Encoding.UTF8.GetBytes(body.ToString()); + if (!string.IsNullOrWhiteSpace(leaseId)) + { + log.LogMessage($"Sending list request: {leaseId} {blockListUploadUrl}"); + req.Headers.Add("x-ms-lease-id", leaseId); + } + req.Headers.Add( + AzureHelper.AuthorizationHeaderString, + AzureHelper.AuthorizationHeader( + AccountName, + AccountKey, + "PUT", + dt1, + req, + string.Empty, + string.Empty, + bodyData.Length.ToString(), + string.Empty)); + + Stream postStream = new MemoryStream(); + postStream.Write(bodyData, 0, bodyData.Length); + postStream.Seek(0, SeekOrigin.Begin); + req.Content = new StreamContent(postStream); + return req; + }; + + using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(log, client, createRequest)) + { + log.LogMessage( + MessageImportance.Low, + "Received response to combine block list for file {0}: Status Code:{1} Status Desc: {2}", + fileName, + response.StatusCode, + await response.Content.ReadAsStringAsync()); + } + } + } + + public async Task FileEqualsExistingBlobAsync( + string accountName, + string accountKey, + string containerName, + string filePath, + string destinationBlob, + int uploadTimeout) + { + using (var client = new HttpClient + { + Timeout = TimeSpan.FromMinutes(uploadTimeout) + }) + { + log.LogMessage( + MessageImportance.Low, + $"Downloading blob {destinationBlob} to check if identical."); + + string blobUrl = AzureHelper.GetBlobRestUrl(accountName, containerName, destinationBlob); + var createRequest = AzureHelper.RequestMessage("GET", blobUrl, accountName, accountKey); + + using (HttpResponseMessage response = await AzureHelper.RequestWithRetry( + log, + client, + createRequest)) + { + if (!response.IsSuccessStatusCode) + { + throw new HttpRequestException( + $"Failed to retrieve existing blob {destinationBlob}, " + + $"status code {response.StatusCode}."); + } + + byte[] existingBytes = await response.Content.ReadAsByteArrayAsync(); + byte[] localBytes = File.ReadAllBytes(filePath); + + bool equal = localBytes.SequenceEqual(existingBytes); + + if (equal) + { + log.LogMessage( + MessageImportance.Normal, + "Item exists in blob storage, and is verified to be identical. " + + $"File: '{filePath}' Blob: '{destinationBlob}'"); + } + + return equal; + } + } + } + + private string DetermineContentTypeBasedOnFileExtension(string filename) + { + if (Path.GetExtension(filename) == ".svg") + { + return "image/svg+xml"; + } + else if (Path.GetExtension(filename) == ".version") + { + return "text/plain"; + } + return string.Empty; + } + private string DetermineCacheControlBasedOnFileExtension(string filename) + { + if (Path.GetExtension(filename) == ".svg") + { + return "No-Cache"; + } + return string.Empty; + } + } +} \ No newline at end of file diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadToAzure.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadToAzure.cs new file mode 100644 index 000000000..5431194d6 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UploadToAzure.cs @@ -0,0 +1,208 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; +using System.Xml; +using ThreadingTask = System.Threading.Tasks.Task; + +namespace Microsoft.DotNet.Build.Tasks +{ + + public class UploadToAzure : AzureConnectionStringBuildTask, ICancelableTask + { + private static readonly CancellationTokenSource TokenSource = new CancellationTokenSource(); + private static readonly CancellationToken CancellationToken = TokenSource.Token; + + /// + /// The name of the container to access. The specified name must be in the correct format, see the + /// following page for more info. https://msdn.microsoft.com/en-us/library/azure/dd135715.aspx + /// + [Required] + public string ContainerName { get; set; } + + /// + /// An item group of files to upload. Each item must have metadata RelativeBlobPath + /// that specifies the path relative to ContainerName where the item will be uploaded. + /// + [Required] + public ITaskItem[] Items { get; set; } + + /// + /// Indicates if the destination blob should be overwritten if it already exists. The default if false. + /// + public bool Overwrite { get; set; } = false; + + /// + /// Enables idempotency when Overwrite is false. + /// + /// false: (default) Attempting to upload an item that already exists fails. + /// + /// true: When an item already exists, download the existing blob to check if it's + /// byte-for-byte identical to the one being uploaded. If so, pass. If not, fail. + /// + public bool PassIfExistingItemIdentical { get; set; } + + /// + /// Specifies the maximum number of clients to concurrently upload blobs to azure + /// + public int MaxClients { get; set; } = 8; + + public int UploadTimeoutInMinutes { get; set; } = 5; + + public void Cancel() + { + TokenSource.Cancel(); + } + + public override bool Execute() + { + return ExecuteAsync(CancellationToken).GetAwaiter().GetResult(); + } + + public async Task ExecuteAsync(CancellationToken ct) + { + ParseConnectionString(); + // If the connection string AND AccountKey & AccountName are provided, error out. + if (Log.HasLoggedErrors) + { + return false; + } + + Log.LogMessage( + MessageImportance.Normal, + "Begin uploading blobs to Azure account {0} in container {1}.", + AccountName, + ContainerName); + + if (Items.Length == 0) + { + Log.LogError("No items were provided for upload."); + return false; + } + + // first check what blobs are present + string checkListUrl = $"{AzureHelper.GetContainerRestUrl(AccountName, ContainerName)}?restype=container&comp=list"; + + HashSet blobsPresent = new HashSet(StringComparer.OrdinalIgnoreCase); + + try + { + using (HttpClient client = new HttpClient()) + { + var createRequest = AzureHelper.RequestMessage("GET", checkListUrl, AccountName, AccountKey); + + Log.LogMessage(MessageImportance.Low, "Sending request to check whether Container blobs exist"); + using (HttpResponseMessage response = await AzureHelper.RequestWithRetry(Log, client, createRequest)) + { + var doc = new XmlDocument(); + doc.LoadXml(await response.Content.ReadAsStringAsync()); + + XmlNodeList nodes = doc.DocumentElement.GetElementsByTagName("Blob"); + + foreach (XmlNode node in nodes) + { + blobsPresent.Add(node["Name"].InnerText); + } + + Log.LogMessage(MessageImportance.Low, "Received response to check whether Container blobs exist"); + } + } + + using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients)) + { + await ThreadingTask.WhenAll(Items.Select(item => UploadAsync(ct, item, blobsPresent, clientThrottle))); + } + + Log.LogMessage(MessageImportance.Normal, "Upload to Azure is complete, a total of {0} items were uploaded.", Items.Length); + } + catch (Exception e) + { + Log.LogErrorFromException(e, true); + } + return !Log.HasLoggedErrors; + } + + private async ThreadingTask UploadAsync(CancellationToken ct, ITaskItem item, HashSet blobsPresent, SemaphoreSlim clientThrottle) + { + if (ct.IsCancellationRequested) + { + Log.LogError("Task UploadToAzure cancelled"); + ct.ThrowIfCancellationRequested(); + } + + string relativeBlobPath = item.GetMetadata("RelativeBlobPath"); + if (string.IsNullOrEmpty(relativeBlobPath)) + throw new Exception(string.Format("Metadata 'RelativeBlobPath' is missing for item '{0}'.", item.ItemSpec)); + + if (!File.Exists(item.ItemSpec)) + throw new Exception(string.Format("The file '{0}' does not exist.", item.ItemSpec)); + + UploadClient uploadClient = new UploadClient(Log); + + if (!Overwrite && blobsPresent.Contains(relativeBlobPath)) + { + if (PassIfExistingItemIdentical && + await ItemEqualsExistingBlobAsync(item, relativeBlobPath, uploadClient, clientThrottle)) + { + return; + } + + throw new Exception(string.Format("The blob '{0}' already exists.", relativeBlobPath)); + } + + string contentType = item.GetMetadata("ContentType"); + + await clientThrottle.WaitAsync(); + + try + { + Log.LogMessage("Uploading {0} to {1}.", item.ItemSpec, ContainerName); + await + uploadClient.UploadBlockBlobAsync( + ct, + AccountName, + AccountKey, + ContainerName, + item.ItemSpec, + relativeBlobPath, + contentType, + UploadTimeoutInMinutes); + } + finally + { + clientThrottle.Release(); + } + } + + private async Task ItemEqualsExistingBlobAsync( + ITaskItem item, + string relativeBlobPath, + UploadClient client, + SemaphoreSlim clientThrottle) + { + await clientThrottle.WaitAsync(); + try + { + return await client.FileEqualsExistingBlobAsync( + AccountName, + AccountKey, + ContainerName, + item.ItemSpec, + relativeBlobPath, + UploadTimeoutInMinutes); + } + finally + { + clientThrottle.Release(); + } + } + } +} \ No newline at end of file diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/AnnotatedUsage.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/AnnotatedUsage.cs new file mode 100644 index 000000000..e15a032f6 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/AnnotatedUsage.cs @@ -0,0 +1,34 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport +{ + public class AnnotatedUsage + { + public Usage Usage { get; set; } + + public string Project { get; set; } + public string SourceBuildPackageIdCreator { get; set; } + public string ProdConPackageIdCreator { get; set; } + public bool EndsUpInOutput { get; set; } + public bool TestProjectByHeuristic { get; set; } + public bool TestProjectOnlyByHeuristic { get; set; } + public bool IsDirectDependency { get; set; } + public bool IsAutoReferenced { get; set; } + + public XElement ToXml() => new XElement( + nameof(AnnotatedUsage), + Usage.ToXml().Attributes(), + Project.ToXAttributeIfNotNull(nameof(Project)), + SourceBuildPackageIdCreator.ToXAttributeIfNotNull(nameof(SourceBuildPackageIdCreator)), + ProdConPackageIdCreator.ToXAttributeIfNotNull(nameof(ProdConPackageIdCreator)), + TestProjectByHeuristic.ToXAttributeIfTrue(nameof(TestProjectByHeuristic)), + TestProjectOnlyByHeuristic.ToXAttributeIfTrue(nameof(TestProjectOnlyByHeuristic)), + IsDirectDependency.ToXAttributeIfTrue(nameof(IsDirectDependency)), + IsAutoReferenced.ToXAttributeIfTrue(nameof(IsAutoReferenced)), + EndsUpInOutput.ToXAttributeIfTrue(nameof(EndsUpInOutput))); + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/Usage.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/Usage.cs new file mode 100644 index 000000000..8b35947db --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/Usage.cs @@ -0,0 +1,112 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using NuGet.Packaging.Core; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport +{ + public class Usage : IEquatable + { + public PackageIdentity PackageIdentity { get; set; } + + public string AssetsFile { get; set; } + + public bool IsDirectDependency { get; set; } + + public bool IsAutoReferenced { get; set; } + + /// + /// The Runtime ID this package is for, or null. Runtime packages (are assumed to) have the + /// id 'runtime.{rid}.{rest of id}'. We can't use a simple regex to grab this value since + /// the RID may have '.' in it, so this is saved in the build context where possible RIDs + /// are read from Microsoft.NETCore.Platforms. + /// + public string RuntimePackageRid { get; set; } + + public XElement ToXml() => new XElement( + nameof(Usage), + PackageIdentity.ToXElement().Attributes(), + AssetsFile.ToXAttributeIfNotNull("File"), + IsDirectDependency.ToXAttributeIfTrue(nameof(IsDirectDependency)), + IsAutoReferenced.ToXAttributeIfTrue(nameof(IsAutoReferenced)), + RuntimePackageRid.ToXAttributeIfNotNull("Rid")); + + public static Usage Parse(XElement xml) => new Usage + { + PackageIdentity = XmlParsingHelpers.ParsePackageIdentity(xml), + AssetsFile = xml.Attribute("File")?.Value, + IsDirectDependency = Convert.ToBoolean(xml.Attribute(nameof(IsDirectDependency))?.Value), + IsAutoReferenced = Convert.ToBoolean(xml.Attribute(nameof(IsAutoReferenced))?.Value), + RuntimePackageRid = xml.Attribute("Rid")?.Value + }; + + public static Usage Create( + string assetsFile, + PackageIdentity identity, + bool isDirectDependency, + bool isAutoReferenced, + IEnumerable possibleRuntimePackageRids) + { + return new Usage + { + AssetsFile = assetsFile, + PackageIdentity = identity, + IsDirectDependency = isDirectDependency, + IsAutoReferenced = isAutoReferenced, + RuntimePackageRid = possibleRuntimePackageRids + .Where(rid => identity.Id.StartsWith($"runtime.{rid}.", StringComparison.Ordinal)) + .OrderByDescending(rid => rid.Length) + .FirstOrDefault() + }; + } + + public PackageIdentity GetIdentityWithoutRid() + { + if (!string.IsNullOrEmpty(RuntimePackageRid)) + { + string prefix = $"runtime.{RuntimePackageRid}."; + if (PackageIdentity.Id.StartsWith(prefix, StringComparison.Ordinal)) + { + return new PackageIdentity( + PackageIdentity.Id + .Remove(0, prefix.Length) + .Insert(0, "runtime.placeholder-rid."), + PackageIdentity.Version); + } + } + return PackageIdentity; + } + + public override string ToString() => + $"{PackageIdentity.Id} {PackageIdentity.Version} " + + (string.IsNullOrEmpty(RuntimePackageRid) ? "" : $"({RuntimePackageRid}) ") + + (string.IsNullOrEmpty(AssetsFile) ? "with unknown use" : $"used by '{AssetsFile}'"); + + public bool Equals(Usage other) + { + if (other is null) + { + return false; + } + if (ReferenceEquals(this, other)) + { + return true; + } + return + Equals(PackageIdentity, other.PackageIdentity) && + string.Equals(AssetsFile, other.AssetsFile, StringComparison.Ordinal) && + string.Equals(RuntimePackageRid, other.RuntimePackageRid, StringComparison.Ordinal); + } + + public override int GetHashCode() => ( + PackageIdentity, + AssetsFile, + RuntimePackageRid + ).GetHashCode(); + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/UsageData.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/UsageData.cs new file mode 100644 index 000000000..ce6c836c4 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/UsageData.cs @@ -0,0 +1,57 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using NuGet.Packaging.Core; +using System.Linq; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport +{ + public class UsageData + { + public string CreatedByRid { get; set; } + public string[] ProjectDirectories { get; set; } + public PackageIdentity[] NeverRestoredTarballPrebuilts { get; set; } + public Usage[] Usages { get; set; } + + public XElement ToXml() => new XElement( + nameof(UsageData), + CreatedByRid == null ? null : new XElement( + nameof(CreatedByRid), + CreatedByRid), + ProjectDirectories?.Any() != true ? null : new XElement( + nameof(ProjectDirectories), + ProjectDirectories + .Select(dir => new XElement("Dir", dir))), + NeverRestoredTarballPrebuilts?.Any() != true ? null : new XElement( + nameof(NeverRestoredTarballPrebuilts), + NeverRestoredTarballPrebuilts + .OrderBy(id => id) + .Select(id => id.ToXElement())), + Usages?.Any() != true ? null : new XElement( + nameof(Usages), + Usages + .OrderBy(u => u.PackageIdentity) + .ThenByOrdinal(u => u.AssetsFile) + .Select(u => u.ToXml()))); + + public static UsageData Parse(XElement xml) => new UsageData + { + CreatedByRid = xml.Element(nameof(CreatedByRid)) + ?.Value, + ProjectDirectories = xml.Element(nameof(ProjectDirectories)) == null ? new string[] { } : + xml.Element(nameof(ProjectDirectories)).Elements() + .Select(x => x.Value) + .ToArray(), + NeverRestoredTarballPrebuilts = xml.Element(nameof(NeverRestoredTarballPrebuilts)) == null ? new PackageIdentity[] { } : + xml.Element(nameof(NeverRestoredTarballPrebuilts)).Elements() + .Select(XmlParsingHelpers.ParsePackageIdentity) + .ToArray(), + Usages = xml.Element(nameof(Usages)) == null ? new Usage[] { } : + xml.Element(nameof(Usages)).Elements() + .Select(Usage.Parse) + .ToArray() + }; + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/ValidateUsageAgainstBaseline.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/ValidateUsageAgainstBaseline.cs new file mode 100644 index 000000000..d96dbf8d0 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/ValidateUsageAgainstBaseline.cs @@ -0,0 +1,141 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Packaging.Core; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport +{ + public class ValidateUsageAgainstBaseline : Task + { + [Required] + public string DataFile { get; set; } + + [Required] + public string BaselineDataFile { get; set; } + + [Required] + public string OutputBaselineFile { get; set; } + + [Required] + public string OutputReportFile { get; set; } + + public bool AllowTestProjectUsage { get; set; } + + public override bool Execute() + { + var used = UsageData.Parse(XElement.Parse(File.ReadAllText(DataFile))); + var baseline = UsageData.Parse(XElement.Parse(File.ReadAllText(BaselineDataFile))); + + Comparison diff = Compare( + used.Usages.Select(u => u.GetIdentityWithoutRid()).Distinct(), + baseline.Usages.Select(u => u.GetIdentityWithoutRid()).Distinct()); + + var report = new XElement("BaselineComparison"); + + bool tellUserToUpdateBaseline = false; + + if (diff.Added.Any()) + { + tellUserToUpdateBaseline = true; + Log.LogError( + $"{diff.Added.Length} new packages used not in baseline! See report " + + $"at {OutputReportFile} for more information. Package IDs are:\n" + + string.Join("\n", diff.Added.Select(u => u.ToString()))); + + // In the report, list full usage info, not only identity. + report.Add( + new XElement( + "New", + used.Usages + .Where(u => diff.Added.Contains(u.GetIdentityWithoutRid())) + .Select(u => u.ToXml()))); + } + if (diff.Removed.Any()) + { + tellUserToUpdateBaseline = true; + Log.LogMessage( + MessageImportance.High, + $"{diff.Removed.Length} packages in baseline weren't used!"); + + report.Add(new XElement("Removed", diff.Removed.Select(id => id.ToXElement()))); + } + if (diff.Unchanged.Any()) + { + Log.LogMessage( + MessageImportance.High, + $"{diff.Unchanged.Length} packages used as expected in the baseline."); + } + + if (!AllowTestProjectUsage) + { + Usage[] testProjectUsages = used.Usages + .Where(WriteUsageReports.IsTestUsageByHeuristic) + .ToArray(); + + if (testProjectUsages.Any()) + { + string[] projects = testProjectUsages + .Select(u => u.AssetsFile) + .Distinct() + .ToArray(); + + Log.LogError( + $"{testProjectUsages.Length} forbidden test usages found in " + + $"{projects.Length} projects:\n" + + string.Join("\n", projects)); + } + } + + // Simplify the used data to what is necessary for a baseline, to reduce file size. + foreach (var usage in used.Usages) + { + usage.AssetsFile = null; + } + used.Usages = used.Usages.Distinct().ToArray(); + + Directory.CreateDirectory(Path.GetDirectoryName(OutputBaselineFile)); + File.WriteAllText(OutputBaselineFile, used.ToXml().ToString()); + + Directory.CreateDirectory(Path.GetDirectoryName(OutputReportFile)); + File.WriteAllText(OutputReportFile, report.ToString()); + + if (tellUserToUpdateBaseline) + { + Log.LogWarning( + "Prebuilt usages are different from the baseline. If detected changes are " + + "acceptable, update baseline with:\n" + + $"cp '{OutputBaselineFile}' '{BaselineDataFile}'"); + } + + return !Log.HasLoggedErrors; + } + + private static Comparison Compare(IEnumerable actual, IEnumerable baseline) + { + return new Comparison(actual.ToArray(), baseline.ToArray()); + } + + private class Comparison + { + public T[] Added { get; } + public T[] Removed { get; } + public T[] Unchanged { get; } + + public Comparison( + IEnumerable actual, + IEnumerable baseline) + { + Added = actual.Except(baseline).ToArray(); + Removed = baseline.Except(actual).ToArray(); + Unchanged = actual.Intersect(baseline).ToArray(); + } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WritePackageUsageData.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WritePackageUsageData.cs new file mode 100644 index 000000000..73c7026a5 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WritePackageUsageData.cs @@ -0,0 +1,284 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NuGet.Packaging.Core; +using NuGet.Versioning; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Threading.Tasks; +using Task = Microsoft.Build.Utilities.Task; + +namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport +{ + public class WritePackageUsageData : Task + { + public string[] RestoredPackageFiles { get; set; } + public string[] TarballPrebuiltPackageFiles { get; set; } + public string[] ReferencePackageFiles { get; set; } + public string[] SourceBuiltPackageFiles { get; set; } + + /// + /// Specific PackageInfo items to check for usage. An alternative to passing lists of nupkgs + /// when the nupkgs have already been parsed to get package info items. + /// + /// %(Identity): Path to the original nupkg. + /// %(PackageId): Identity of the package. + /// %(PackageVersion): Version of the package. + /// + public ITaskItem[] NuGetPackageInfos { get; set; } + + /// + /// runtime.json files (from Microsoft.NETCore.Platforms) to use to look for the set of all + /// possible runtimes. This is used to determine which part of a package id is its + /// 'runtime.{rid}.' prefix, if it has the prefix. + /// + public string[] PlatformsRuntimeJsonFiles { get; set; } + + /// + /// Keep track of the RID built that caused these usages. + /// + public string TargetRid { get; set; } + + /// + /// Project directories to scan for project.assets.json files. If these directories contain + /// one another, the project.assets.json files is reported as belonging to the first project + /// directory that contains it. For useful results, put the leafmost directories first. + /// + /// This isn't used here, but it's included in the usage data so report generation can + /// happen independently of commits that add/remove submodules. + /// + public string[] ProjectDirectories { get; set; } + + /// + /// A root dir that contains all ProjectDirectories. This is used to find the relative path + /// of each usage. + /// + [Required] + public string RootDir { get; set; } + + /// + /// project.assets.json files to ignore, for example, because they are checked-in assets not + /// generated during source-build and cause false positives. + /// + public string[] IgnoredProjectAssetsJsonFiles { get; set; } + + /// + /// Output usage data JSON file path. + /// + [Required] + public string DataFile { get; set; } + + /// + /// If passed, the path of the archive file to generate that includes a copy of all + /// project.asset.json files found. + /// + public string ProjectAssetsJsonArchiveFile { get; set; } + + public override bool Execute() + { + DateTime startTime = DateTime.Now; + Log.LogMessage(MessageImportance.High, "Writing package usage data..."); + + string[] projectDirectoriesOutsideRoot = ProjectDirectories.NullAsEmpty() + .Where(dir => !dir.StartsWith(RootDir, StringComparison.Ordinal)) + .ToArray(); + + if (projectDirectoriesOutsideRoot.Any()) + { + throw new ArgumentException( + $"All ProjectDirectories must be in RootDir '{RootDir}', but found " + + string.Join(", ", projectDirectoriesOutsideRoot)); + } + + Log.LogMessage(MessageImportance.Low, "Finding set of RIDs..."); + + string[] possibleRids = PlatformsRuntimeJsonFiles.NullAsEmpty() + .SelectMany(ReadRidsFromRuntimeJson) + .Distinct() + .ToArray(); + + Log.LogMessage(MessageImportance.Low, "Reading package identities..."); + + PackageIdentity[] restored = RestoredPackageFiles.NullAsEmpty() + .Select(ReadNuGetPackageInfos.ReadIdentity) + .Distinct() + .ToArray(); + + PackageIdentity[] tarballPrebuilt = TarballPrebuiltPackageFiles.NullAsEmpty() + .Select(ReadNuGetPackageInfos.ReadIdentity) + .Distinct() + .ToArray(); + + PackageIdentity[] referencePackages = ReferencePackageFiles.NullAsEmpty() + .Select(ReadNuGetPackageInfos.ReadIdentity) + .Distinct() + .ToArray(); + + PackageIdentity[] sourceBuilt = SourceBuiltPackageFiles.NullAsEmpty() + .Select(ReadNuGetPackageInfos.ReadIdentity) + .Distinct() + .ToArray(); + + IEnumerable prebuilt = restored.Except(sourceBuilt).Except(referencePackages); + + PackageIdentity[] toCheck = NuGetPackageInfos.NullAsEmpty() + .Select(item => new PackageIdentity( + item.GetMetadata("PackageId"), + NuGetVersion.Parse(item.GetMetadata("PackageVersion")))) + .Concat(prebuilt) + .ToArray(); + + Log.LogMessage(MessageImportance.Low, "Finding project.assets.json files..."); + + string[] assetFiles = Directory + .GetFiles(RootDir, "project.assets.json", SearchOption.AllDirectories) + .Select(GetPathRelativeToRoot) + .Except(IgnoredProjectAssetsJsonFiles.NullAsEmpty().Select(GetPathRelativeToRoot)) + .ToArray(); + + if (!string.IsNullOrEmpty(ProjectAssetsJsonArchiveFile)) + { + Log.LogMessage(MessageImportance.Low, "Archiving project.assets.json files..."); + + Directory.CreateDirectory(Path.GetDirectoryName(ProjectAssetsJsonArchiveFile)); + + using (var projectAssetArchive = new ZipArchive( + File.Open( + ProjectAssetsJsonArchiveFile, + FileMode.Create, + FileAccess.ReadWrite), + ZipArchiveMode.Create)) + { + // Only one entry can be open at a time, so don't do this during the Parallel + // ForEach later. + foreach (var relativePath in assetFiles) + { + using (var stream = File.OpenRead(Path.Combine(RootDir, relativePath))) + using (Stream entryWriter = projectAssetArchive + .CreateEntry(relativePath, CompressionLevel.Optimal) + .Open()) + { + stream.CopyTo(entryWriter); + } + } + } + } + + Log.LogMessage(MessageImportance.Low, "Reading usage info..."); + + var usages = new ConcurrentBag(); + + Parallel.ForEach( + assetFiles, + assetFile => + { + JObject jObj; + + using (var file = File.OpenRead(Path.Combine(RootDir, assetFile))) + using (var reader = new StreamReader(file)) + using (var jsonReader = new JsonTextReader(reader)) + { + jObj = (JObject)JToken.ReadFrom(jsonReader); + } + + var properties = new HashSet( + jObj.SelectTokens("$.targets.*").Children() + .Concat(jObj.SelectToken("$.libraries")) + .Select(t => ((JProperty)t).Name) + .Distinct(), + StringComparer.OrdinalIgnoreCase); + + var directDependencies = jObj.SelectTokens("$.project.frameworks.*.dependencies").Children().Select(dep => + new + { + name = ((JProperty)dep).Name, + target = dep.SelectToken("$..target")?.ToString(), + version = VersionRange.Parse(dep.SelectToken("$..version")?.ToString()), + autoReferenced = dep.SelectToken("$..autoReferenced")?.ToString() == "True", + }) + .ToArray(); + + foreach (var identity in toCheck + .Where(id => properties.Contains(id.Id + "/" + id.Version.OriginalVersion))) + { + var directDependency = + directDependencies?.FirstOrDefault( + d => d.name == identity.Id && + d.version.Satisfies(identity.Version)); + usages.Add(Usage.Create( + assetFile, + identity, + directDependency != null, + directDependency?.autoReferenced == true, + possibleRids)); + } + }); + + Log.LogMessage(MessageImportance.Low, "Searching for unused packages..."); + + foreach (PackageIdentity restoredWithoutUsagesFound in + toCheck.Except(usages.Select(u => u.PackageIdentity))) + { + usages.Add(Usage.Create( + null, + restoredWithoutUsagesFound, + false, + false, + possibleRids)); + } + + // Packages that were included in the tarball as prebuilts, but weren't even restored. + PackageIdentity[] neverRestoredTarballPrebuilts = tarballPrebuilt + .Except(restored) + .ToArray(); + + Log.LogMessage(MessageImportance.Low, $"Writing data to '{DataFile}'..."); + + var data = new UsageData + { + CreatedByRid = TargetRid, + Usages = usages.ToArray(), + NeverRestoredTarballPrebuilts = neverRestoredTarballPrebuilts, + ProjectDirectories = ProjectDirectories + ?.Select(GetPathRelativeToRoot) + .ToArray() + }; + + Directory.CreateDirectory(Path.GetDirectoryName(DataFile)); + File.WriteAllText(DataFile, data.ToXml().ToString()); + + Log.LogMessage( + MessageImportance.High, + $"Writing package usage data... done. Took {DateTime.Now - startTime}"); + + return !Log.HasLoggedErrors; + } + + private string GetPathRelativeToRoot(string path) + { + if (path.StartsWith(RootDir)) + { + return path.Substring(RootDir.Length).Replace(Path.DirectorySeparatorChar, '/'); + } + + throw new ArgumentException($"Path '{path}' is not within RootDir '{RootDir}'"); + } + + private static string[] ReadRidsFromRuntimeJson(string path) + { + var root = JObject.Parse(File.ReadAllText(path)); + return root["runtimes"] + .Values() + .Select(o => o.Name) + .ToArray(); + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WriteUsageBurndownData.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WriteUsageBurndownData.cs new file mode 100644 index 000000000..ab40fa590 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WriteUsageBurndownData.cs @@ -0,0 +1,130 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Newtonsoft.Json; +using Newtonsoft.Json.Linq; +using NuGet.Packaging.Core; +using NuGet.Versioning; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.IO.Compression; +using System.Linq; +using System.Net; +using System.Threading.Tasks; +using System.Xml.Linq; +using Task = Microsoft.Build.Utilities.Task; + +namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport +{ + public class WriteUsageBurndownData : Task + { + /// + /// Specifies the root directory for git. + /// Note: Requires a trailing "/" when specifying the directory. + /// + [Required] + public string RootDirectory { get; set; } + + /// + /// Specifies the path to the prebuilt baseline file + /// to be used to generate the burndown. + /// + [Required] + public string PrebuiltBaselineFile { get; set; } + + /// + /// Output data CSV file. + /// + [Required] + public string OutputFilePath { get; set; } + + public override bool Execute() + { + string baselineRelativeFileName = PrebuiltBaselineFile.Replace(RootDirectory, ""); + string gitLogCommand = $"log --first-parent --pretty=format:%H,%f,%ci -- {PrebuiltBaselineFile}"; + + DateTime startTime = DateTime.Now; + Log.LogMessage(MessageImportance.High, "Generating summary usage burndown data..."); + + ParallelQuery data = ExecuteGitCommand(RootDirectory, gitLogCommand).AsParallel().Select(commitLine => + { + var splitLine = commitLine.Split(','); + var commit = new Commit() + { + Sha = splitLine[0], + Title = splitLine[1], + CommitDate = DateTime.Parse(splitLine[2]) + }; + string fileContents = GetFileContents(baselineRelativeFileName, commit.Sha); + Usage[] usages = UsageData.Parse(XElement.Parse(fileContents)).Usages.NullAsEmpty().ToArray(); + commit.PackageVersionCount = usages.Count(); + commit.PackageCount = usages.GroupBy(i => i.PackageIdentity.Id).Select(grp => grp.First()).Count(); + return commit; + }) + .Select(c => c.ToString()); + + Directory.CreateDirectory(Path.GetDirectoryName(OutputFilePath)); + + File.WriteAllLines(OutputFilePath, data); + + Log.LogMessage( + MessageImportance.High, + $"Generating summary usage burndown data at {OutputFilePath} done. Took {DateTime.Now - startTime}"); + + return !Log.HasLoggedErrors; + } + + /// + /// Get the contents of a git file based on the commit sha. + /// + /// The relative path (from the git root) to the file. + /// The commit sha for the version of the file to get. + /// The contents of the specified file. + private string GetFileContents(string relativeFilePath, string commitSha) + { + WebClient client = new WebClient(); + var xmlString = client.DownloadString($"https://raw.githubusercontent.com/dotnet/source-build/{commitSha}/{relativeFilePath.Replace('\\', '/')}"); + return xmlString; + } + + /// + /// Executes a git command and returns the result. + /// + /// The working directory for the git command. + /// The git command to execute. + /// An array of the output lines of the git command. + private string[] ExecuteGitCommand(string workingDirectory, string command) + { + string[] returnData; + Process _process = new Process(); + _process.StartInfo.FileName = "git"; + _process.StartInfo.Arguments = command; + _process.StartInfo.WorkingDirectory = workingDirectory; + _process.StartInfo.RedirectStandardOutput = true; + _process.StartInfo.UseShellExecute = false; + _process.Start(); + returnData = _process.StandardOutput.ReadToEnd().Split('\n'); + _process.WaitForExit(); + return returnData; + } + + private class Commit + { + public string Sha { get; set; } + public string Title { get; set; } + public DateTime CommitDate { get; set; } + public int PackageVersionCount { get; set; } + public int PackageCount { get; set; } + + public override string ToString() + { + return $"{Sha}, {Title}, {CommitDate}, {PackageVersionCount}, {PackageCount}"; + } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WriteUsageReports.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WriteUsageReports.cs new file mode 100644 index 000000000..e236d16a3 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/WriteUsageReports.cs @@ -0,0 +1,303 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using Microsoft.DotNet.Build.Tasks; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport +{ + public class WriteUsageReports : Task + { + private const string SnapshotPrefix = "PackageVersions.props.pre."; + private const string SnapshotSuffix = ".xml"; + + /// + /// Source usage data JSON file. + /// + [Required] + public string DataFile { get; set; } + + /// + /// A set of "PackageVersions.props.pre.{repo}.xml" files. They are analyzed to find + /// packages built during source-build, and which repo built them. This info is added to the + /// report. New packages are associated to a repo by going through each PVP in ascending + /// file modification order. + /// + public ITaskItem[] PackageVersionPropsSnapshots { get; set; } + + /// + /// Infos that associate packages to the ProdCon build they're from. + /// + /// %(PackageId): Identity of the package. + /// %(OriginBuildName): Name of the build that produced this package. + /// + public ITaskItem[] ProdConPackageInfos { get; set; } + + /// + /// Path to a ProdCon build manifest file (build.xml) as an alternative way to pass + /// ProdConPackageInfos items. + /// + public string ProdConBuildManifestFile { get; set; } + + /// + /// File containing the results of poisoning the prebuilts. Example format: + /// + /// MATCH: output built\dotnet-sdk-...\System.Collections.dll(hash 4b...31) matches one of: + /// intermediate\netstandard.library.2.0.1.nupkg\build\...\System.Collections.dll + /// + /// The usage report reads this file, looking for 'intermediate\*.nupkg' to annotate. + /// + public string PoisonedReportFile { get; set; } + + [Required] + public string OutputDirectory { get; set; } + + public override bool Execute() + { + UsageData data = UsageData.Parse(XElement.Parse(File.ReadAllText(DataFile))); + + IEnumerable sourceBuildRepoOutputs = GetSourceBuildRepoOutputs(); + + // Map package id to the build name that created them in a ProdCon build. + var prodConPackageOrigin = new Dictionary( + StringComparer.OrdinalIgnoreCase); + + foreach (ITaskItem item in ProdConPackageInfos.NullAsEmpty()) + { + AddProdConPackage( + prodConPackageOrigin, + item.GetMetadata("PackageId"), + item.GetMetadata("OriginBuildName")); + } + + if (File.Exists(ProdConBuildManifestFile)) + { + var xml = XElement.Parse(File.ReadAllText(ProdConBuildManifestFile)); + foreach (var x in xml.Descendants("Package")) + { + AddProdConPackage( + prodConPackageOrigin, + x.Attribute("Id")?.Value, + x.Attribute("OriginBuildName")?.Value); + } + } + + var poisonNupkgFilenames = new HashSet(StringComparer.OrdinalIgnoreCase); + + if (File.Exists(PoisonedReportFile)) + { + foreach (string line in File.ReadAllLines(PoisonedReportFile)) + { + string[] segments = line.Split('\\'); + if (segments.Length > 2 && + segments[0].Trim() == "intermediate" && + segments[1].EndsWith(".nupkg")) + { + poisonNupkgFilenames.Add(Path.GetFileNameWithoutExtension(segments[1])); + } + } + } + + var report = new XElement("AnnotatedUsages"); + + var annotatedUsages = data.Usages.NullAsEmpty() + .Select(usage => + { + string id = usage.PackageIdentity.Id; + string version = usage.PackageIdentity.Version.OriginalVersion; + + string pvpIdent = WriteBuildOutputProps.GetPropertyName(id); + + var sourceBuildCreator = new StringBuilder(); + foreach (RepoOutput output in sourceBuildRepoOutputs) + { + foreach (PackageVersionPropsElement p in output.Built) + { + if (p.Name.Equals(pvpIdent, StringComparison.OrdinalIgnoreCase)) + { + if (sourceBuildCreator.Length != 0) + { + sourceBuildCreator.Append(" "); + } + sourceBuildCreator.Append(output.Repo); + sourceBuildCreator.Append(" "); + sourceBuildCreator.Append(p.Name); + sourceBuildCreator.Append("/"); + sourceBuildCreator.Append(p.Version); + } + } + } + + prodConPackageOrigin.TryGetValue(id, out string prodConCreator); + + return new AnnotatedUsage + { + Usage = usage, + + Project = data.ProjectDirectories + ?.FirstOrDefault(p => usage.AssetsFile?.StartsWith(p) ?? false), + + SourceBuildPackageIdCreator = sourceBuildCreator.Length == 0 + ? null + : sourceBuildCreator.ToString(), + + ProdConPackageIdCreator = prodConCreator, + + TestProjectByHeuristic = IsTestUsageByHeuristic(usage), + + EndsUpInOutput = poisonNupkgFilenames.Contains($"{id}.{version}") + }; + }) + .ToArray(); + + foreach (var onlyTestProjectUsage in annotatedUsages + .GroupBy(u => u.Usage.PackageIdentity) + .Where(g => g.All(u => u.TestProjectByHeuristic)) + .SelectMany(g => g)) + { + onlyTestProjectUsage.TestProjectOnlyByHeuristic = true; + } + + report.Add(annotatedUsages.Select(u => u.ToXml())); + + Directory.CreateDirectory(OutputDirectory); + + File.WriteAllText( + Path.Combine(OutputDirectory, "annotated-usage.xml"), + report.ToString()); + + return !Log.HasLoggedErrors; + } + + private RepoOutput[] GetSourceBuildRepoOutputs() + { + var pvpSnapshotFiles = PackageVersionPropsSnapshots.NullAsEmpty() + .Select(item => + { + var content = File.ReadAllText(item.ItemSpec); + return new + { + Path = item.ItemSpec, + Content = content, + Xml = XElement.Parse(content) + }; + }) + .OrderBy(snapshot => + { + // Get the embedded creation time if possible: the file's original metadata may + // have been destroyed by copying, zipping, etc. + string creationTime = snapshot.Xml + // Get the second PropertyGroup. + .Elements().Skip(1).FirstOrDefault() + // Get the creation time element. + ?.Element(snapshot.Xml + .GetDefaultNamespace() + .GetName(WriteBuildOutputProps.CreationTimePropertyName)) + ?.Value; + + if (string.IsNullOrEmpty(creationTime)) + { + Log.LogError($"No creation time property found in snapshot {snapshot.Path}"); + return default(DateTime); + } + + return new DateTime(long.Parse(creationTime)); + }) + .Select(snapshot => + { + string filename = Path.GetFileName(snapshot.Path); + return new + { + Repo = filename.Substring( + SnapshotPrefix.Length, + filename.Length - SnapshotPrefix.Length - SnapshotSuffix.Length), + PackageVersionProp = PackageVersionPropsElement.Parse(snapshot.Xml) + }; + }) + .ToArray(); + + return pvpSnapshotFiles.Skip(1) + .Zip(pvpSnapshotFiles, (pvp, prev) => new RepoOutput + { + Repo = prev.Repo, + Built = pvp.PackageVersionProp.Except(prev.PackageVersionProp).ToArray() + }) + .ToArray(); + } + + private void AddProdConPackage( + Dictionary packageOrigin, + string id, + string origin) + { + if (!string.IsNullOrEmpty(id) && + !string.IsNullOrEmpty(origin)) + { + packageOrigin[id] = origin; + } + } + + public static bool IsTestUsageByHeuristic(Usage usage) + { + string[] assetsFileParts = usage.AssetsFile?.Split('/', '\\'); + + // If the dir name ends in Test(s), it's probably a test project. + // Ignore the first two segments to avoid classifying everything in "src/vstest". + // This also catches "test" dirs that contain many test projects. + if (assetsFileParts?.Skip(2).Any(p => + p.EndsWith("Tests", StringComparison.OrdinalIgnoreCase) || + p.EndsWith("Test", StringComparison.OrdinalIgnoreCase)) == true) + { + return true; + } + + // CoreFX restores test dependencies during this sync project. + if (assetsFileParts?.Contains("XUnit.Runtime") == true) + { + return true; + } + + return false; + } + + private class RepoOutput + { + public string Repo { get; set; } + public PackageVersionPropsElement[] Built { get; set; } + } + + private struct PackageVersionPropsElement + { + public static PackageVersionPropsElement[] Parse(XElement xml) + { + return xml + // Get the single PropertyGroup + .Elements() + .First() + // Get all *PackageVersion property elements. + .Elements() + .Select(x => new PackageVersionPropsElement( + x.Name.LocalName, + x.Nodes().OfType().First().Value)) + .ToArray(); + } + + public string Name { get; } + public string Version { get; } + + public PackageVersionPropsElement(string name, string version) + { + Name = name; + Version = version; + } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/XmlParsingHelpers.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/XmlParsingHelpers.cs new file mode 100644 index 000000000..c30173fff --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/UsageReport/XmlParsingHelpers.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using NuGet.Packaging.Core; +using NuGet.Versioning; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Xml.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport +{ + internal static class XmlParsingHelpers + { + public static XElement ToXElement(this PackageIdentity ident) => new XElement( + "PackageIdentity", + new XAttribute("Id", ident.Id), + new XAttribute("Version", ident.Version.OriginalVersion)); + + public static XAttribute ToXAttributeIfNotNull(this object value, string name) => + value == null ? null : new XAttribute(name, value); + + public static XAttribute ToXAttributeIfTrue(this bool value, string name) => + value == false ? null : new XAttribute(name, value); + + public static PackageIdentity ParsePackageIdentity(XElement xml) => new PackageIdentity( + xml.Attribute("Id").Value, + NuGetVersion.Parse(xml.Attribute("Version").Value)); + + public static IOrderedEnumerable OrderByOrdinal( + this IEnumerable source, + Func selector) + { + return source.OrderBy(selector, StringComparer.Ordinal); + } + + public static IOrderedEnumerable ThenByOrdinal( + this IOrderedEnumerable source, + Func selector) + { + return source.ThenBy(selector, StringComparer.Ordinal); + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteBuildOutputProps.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteBuildOutputProps.cs new file mode 100644 index 000000000..fb1d2fe4d --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteBuildOutputProps.cs @@ -0,0 +1,142 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using NuGet.Versioning; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class WriteBuildOutputProps : Task + { + private static readonly Regex InvalidElementNameCharRegex = new Regex(@"(^|[^A-Za-z0-9])(?.)"); + + public const string CreationTimePropertyName = "BuildOutputPropsCreationTime"; + + [Required] + public ITaskItem[] NuGetPackages { get; set; } + + [Required] + public string OutputPath { get; set; } + + /// + /// Adds a second PropertyGroup to the output XML containing a property with the time of + /// creation in UTC DateTime Ticks. This can be used to track creation time in situations + /// where file metadata isn't reliable or preserved. + /// + public bool IncludeCreationTimeProperty { get; set; } + + /// + /// Properties to add to the build output props, which may not exist as nupkgs. + /// FOr example, this is used to pass the version of the CLI toolset archives. + /// + /// %(Identity): Package identity. + /// %(Version): Package version. + /// + public ITaskItem[] ExtraProperties { get; set; } + + /// + /// Additional assets to be added to the build output props. + /// i.e. /bin/obj/x64/Release/blobs/Toolset/3.0.100 + /// This parameter is the / portion only, and the asset + /// must be in a / folder. + /// + public string[] AdditionalAssetDirs { get; set; } + + public override bool Execute() + { + PackageIdentity[] latestPackages = NuGetPackages + .Select(item => + { + using (var reader = new PackageArchiveReader(item.GetMetadata("FullPath"))) + { + return reader.GetIdentity(); + } + }) + .GroupBy(identity => identity.Id) + .Select(g => g.OrderBy(id => id.Version).Last()) + .OrderBy(id => id.Id) + .ToArray(); + + var additionalAssets = (AdditionalAssetDirs ?? new string[0]) + .Where(Directory.Exists) + .Where(dir => Directory.GetDirectories(dir).Count() > 0) + .Select(dir => new { + Name = new DirectoryInfo(dir).Name + "Version", + Version = new DirectoryInfo(Directory.EnumerateDirectories(dir).OrderBy(s => s).Last()).Name + }).ToArray(); + + Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); + + using (var outStream = File.Open(OutputPath, FileMode.Create)) + using (var sw = new StreamWriter(outStream, new UTF8Encoding(false))) + { + sw.WriteLine(@""); + sw.WriteLine(@""); + sw.WriteLine(@" "); + foreach (PackageIdentity packageIdentity in latestPackages) + { + string propertyName = GetPropertyName(packageIdentity.Id); + sw.WriteLine($" <{propertyName}>{packageIdentity.Version}"); + + propertyName = GetAlternatePropertyName(packageIdentity.Id); + sw.WriteLine($" <{propertyName}>{packageIdentity.Version}"); + } + foreach (var extraProp in ExtraProperties ?? Enumerable.Empty()) + { + string propertyName = extraProp.GetMetadata("Identity"); + bool doNotOverwrite = false; + string overwriteCondition = string.Empty; + if (bool.TryParse(extraProp.GetMetadata("DoNotOverwrite"), out doNotOverwrite) && doNotOverwrite) + { + overwriteCondition = $" Condition=\"'$({propertyName})' == ''\""; + } + sw.WriteLine($" <{propertyName}{overwriteCondition}>{extraProp.GetMetadata("Version")}"); + } + foreach (var additionalAsset in additionalAssets) + { + sw.WriteLine($" <{additionalAsset.Name}>{additionalAsset.Version}"); + } + sw.WriteLine(@" "); + if (IncludeCreationTimeProperty) + { + sw.WriteLine(@" "); + sw.WriteLine($@" <{CreationTimePropertyName}>{DateTime.UtcNow.Ticks}"); + sw.WriteLine(@" "); + } + sw.WriteLine(@""); + } + + return true; + } + + public static string GetPropertyName(string id) + { + string formattedId = InvalidElementNameCharRegex.Replace( + id, + match => match.Groups?["FirstPartChar"].Value.ToUpperInvariant() + ?? string.Empty); + + return $"{formattedId}PackageVersion"; + } + + public static string GetAlternatePropertyName(string id) + { + string formattedId = InvalidElementNameCharRegex.Replace( + id, + match => match.Groups?["FirstPartChar"].Value.ToUpperInvariant() + ?? string.Empty); + + return $"{formattedId}Version"; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceAndVersionProps.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceAndVersionProps.cs new file mode 100644 index 000000000..ee8a9ba7d --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceAndVersionProps.cs @@ -0,0 +1,156 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using NuGet.Versioning; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class WriteRestoreSourceAndVersionProps : Task + { + private static readonly Regex InvalidElementNameCharRegex = new Regex(@"(^|[^A-Za-z0-9])(?.)"); + + public const string CreationTimePropertyName = "BuildOutputPropsCreationTime"; + + [Required] + public ITaskItem[] RestoreSources { get; set; } + + [Required] + public ITaskItem[] NuGetPackages { get; set; } + + [Required] + public string OutputPath { get; set; } + + /// + /// Adds a second PropertyGroup to the output XML containing a property with the time of + /// creation in UTC DateTime Ticks. This can be used to track creation time in situations + /// where file metadata isn't reliable or preserved. + /// + public bool IncludeCreationTimeProperty { get; set; } + + /// + /// Properties to add to the build output props, which may not exist as nupkgs. + /// FOr example, this is used to pass the version of the CLI toolset archives. + /// + /// %(Identity): Package identity. + /// %(Version): Package version. + /// + public ITaskItem[] ExtraProperties { get; set; } + + /// + /// Additional assets to be added to the build output props. + /// i.e. /bin/obj/x64/Release/blobs/Toolset/3.0.100 + /// This parameter is the / portion only, and the asset + /// must be in a / folder. + /// + public string[] AdditionalAssetDirs { get; set; } + + public override bool Execute() + { + PackageIdentity[] latestPackages = NuGetPackages + .Select(item => + { + using (var reader = new PackageArchiveReader(item.GetMetadata("FullPath"))) + { + return reader.GetIdentity(); + } + }) + .GroupBy(identity => identity.Id) + .Select(g => g.OrderBy(id => id.Version).Last()) + .OrderBy(id => id.Id) + .ToArray(); + + var additionalAssets = (AdditionalAssetDirs ?? new string[0]) + .Where(Directory.Exists) + .Where(dir => Directory.GetDirectories(dir).Count() > 0) + .Select(dir => new { + Name = new DirectoryInfo(dir).Name + "Version", + Version = new DirectoryInfo(Directory.EnumerateDirectories(dir).OrderBy(s => s).Last()).Name + }).ToArray(); + + Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); + + using (var outStream = File.Open(OutputPath, FileMode.Create)) + using (var sw = new StreamWriter(outStream, new UTF8Encoding(false))) + { + sw.WriteLine(@""); + sw.WriteLine(@""); + sw.WriteLine(@" "); + sw.WriteLine(@" "); + foreach (ITaskItem restoreSourceItem in RestoreSources) + { + sw.WriteLine($" {restoreSourceItem.ItemSpec};"); + } + sw.WriteLine(@" "); + sw.WriteLine(@" "); + + sw.WriteLine(@" "); + foreach (PackageIdentity packageIdentity in latestPackages) + { + string propertyName = GetPropertyName(packageIdentity.Id); + string shortPropertyName = GetShortPropertyName(packageIdentity.Id); + + sw.WriteLine($" <{propertyName}>{packageIdentity.Version}"); + sw.WriteLine($" <{shortPropertyName}>{packageIdentity.Version}"); + } + foreach (var extraProp in ExtraProperties ?? Enumerable.Empty()) + { + string propertyName = extraProp.GetMetadata("Identity"); + bool doNotOverwrite = false; + string overwriteCondition = string.Empty; + if (bool.TryParse(extraProp.GetMetadata("DoNotOverwrite"), out doNotOverwrite) && doNotOverwrite) + { + overwriteCondition = $" Condition=\"'$({propertyName})' == ''\""; + } + sw.WriteLine($" <{propertyName}{overwriteCondition}>{extraProp.GetMetadata("Version")}"); + } + foreach (var additionalAsset in additionalAssets) + { + sw.WriteLine($" <{additionalAsset.Name}>{additionalAsset.Version}"); + } + sw.WriteLine(@" "); + + if (IncludeCreationTimeProperty) + { + sw.WriteLine(@" "); + sw.WriteLine($@" <{CreationTimePropertyName}>{DateTime.UtcNow.Ticks}"); + sw.WriteLine(@" "); + } + + sw.WriteLine(@""); + } + + return true; + } + + public static string GetPropertyName(string id) + { + string formattedId = InvalidElementNameCharRegex.Replace( + id, + match => match.Groups?["FirstPartChar"].Value.ToUpperInvariant() + ?? string.Empty); + + return $"{formattedId}PackageVersion"; + } + + public static string GetShortPropertyName(string id) + { + string formattedId = InvalidElementNameCharRegex.Replace( + id, + match => match.Groups?["FirstPartChar"].Value.ToUpperInvariant() + ?? string.Empty); + + return $"{formattedId}Version"; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceProps.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceProps.cs new file mode 100644 index 000000000..8e983531b --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceProps.cs @@ -0,0 +1,43 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using System.IO; +using System.Text; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class WriteRestoreSourceProps : Task + { + [Required] + public ITaskItem[] RestoreSources { get; set; } + + [Required] + public string OutputPath { get; set; } + + public override bool Execute() + { + Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); + + using (var outStream = File.Open(OutputPath, FileMode.Create)) + using (var sw = new StreamWriter(outStream, new UTF8Encoding(false))) + { + sw.WriteLine(@""); + sw.WriteLine(@""); + sw.WriteLine(@" "); + sw.WriteLine(@" "); + foreach (ITaskItem restoreSourceItem in RestoreSources) + { + sw.WriteLine($" {restoreSourceItem.ItemSpec};"); + } + sw.WriteLine(@" "); + sw.WriteLine(@" "); + sw.WriteLine(@""); + } + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteSourceRepoProperties.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteSourceRepoProperties.cs new file mode 100644 index 000000000..5b96f9f87 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteSourceRepoProperties.cs @@ -0,0 +1,317 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Tasks; +using Microsoft.Build.Utilities; +using Microsoft.DotNet.SourceBuild.Tasks.Models; +using NuGet.Versioning; +using System; +using System.Collections.Generic; +using System.Globalization; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml.Serialization; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class WriteSourceRepoProperties : Task + { + [Required] + public string VersionDetailsFile { get; set; } + + [Required] + public string ClonedSubmoduleGitRootDirectory { get; set; } + + [Required] + public string ClonedSubmoduleDirectory { get; set; } + + [Required] + public string SourceBuildMetadataDir { get; set; } + + public override bool Execute() + { + var serializer = new XmlSerializer(typeof(VersionDetails)); + + VersionDetails versionDetails = null; + using (var stream = File.OpenRead(VersionDetailsFile)) + { + versionDetails = (VersionDetails)serializer.Deserialize(stream); + } + + var allRepoProps = new Dictionary(); + + foreach (var dep in versionDetails.ToolsetDependencies.Concat(versionDetails.ProductDependencies)) + { + Log.LogMessage(MessageImportance.Normal, $"[{DateTimeOffset.Now}] Starting dependency {dep.ToString()}"); + string repoPath = DeriveRepoPath(ClonedSubmoduleDirectory, dep.Uri, dep.Sha); + string repoGitDir = DeriveRepoGitDirPath(ClonedSubmoduleGitRootDirectory, dep.Uri); + if (Directory.Exists(repoGitDir)) + { + foreach (string repoName in GetRepoNamesOrDefault(dep)) + { + string safeRepoName = repoName.Replace("-", ""); + try + { + WriteMinimalMetadata(repoPath, dep.Uri, dep.Sha); + WriteSourceBuildMetadata(SourceBuildMetadataDir, repoGitDir, dep); + if (File.Exists(Path.Combine(repoPath, ".gitmodules"))) + { + HandleSubmodules(repoPath, repoGitDir, dep); + } + allRepoProps[$"{safeRepoName}GitCommitHash"] = dep.Sha; + allRepoProps[$"{safeRepoName}OutputPackageVersion"] = dep.Version; + } + catch (Exception e) + { + Log.LogErrorFromException(e, true, true, null); + } + } + } + else + { + Log.LogMessage(MessageImportance.Normal, $"[{DateTimeOffset.Now}] Skipping dependency {dep.ToString()} - git dir {repoGitDir} doesn't exist"); + } + } + string allRepoPropsPath = Path.Combine(SourceBuildMetadataDir, "AllRepoVersions.props"); + Log.LogMessage(MessageImportance.Normal, $"[{DateTimeOffset.Now}] Writing all repo versions to {allRepoPropsPath}"); + WritePropsFile(allRepoPropsPath, allRepoProps); + + return !Log.HasLoggedErrors; + } + + private void WriteSourceBuildMetadata(string sourceBuildMetadataPath, string repoGitDir, Dependency dependency) + { + foreach (string repoName in GetRepoNamesOrDefault(dependency)) + { + string propsPath = Path.Combine(sourceBuildMetadataPath, $"{repoName}.props"); + string commitCount = GetCommitCount(repoGitDir, dependency.Sha); + DerivedVersion derivedVersion = GetVersionInfo(dependency.Version, commitCount); + var repoProps = new Dictionary + { + ["GitCommitHash"] = dependency.Sha, + ["GitCommitCount"] = commitCount, + ["GitCommitDate"] = GetCommitDate(repoGitDir, dependency.Sha), + ["OfficialBuildId"] = derivedVersion.OfficialBuildId, + ["OutputPackageVersion"] = dependency.Version, + ["PreReleaseVersionLabel"] = derivedVersion.PreReleaseVersionLabel, + ["IsStable"] = string.IsNullOrWhiteSpace(derivedVersion.PreReleaseVersionLabel) ? "true" : "false", + }; + WritePropsFile(propsPath, repoProps); + } + } + + + /// + /// Reverse a version in the Arcade style (https://github.com/dotnet/arcade/blob/fb92b14d8cd07cf44f8f7eefa8ac58d7ffd05f3f/src/Microsoft.DotNet.Arcade.Sdk/tools/Version.BeforeCommonTargets.targets#L18) + /// back to an OfficialBuildId + ReleaseLabel which we can then supply to get the same resulting version number. + /// + /// The complete version, e.g. 1.0.0-beta1-19720.5 + /// The current commit count of the repo. This is used for some repos that do not use the standard versioning scheme. + /// + private static DerivedVersion GetVersionInfo(string version, string commitCount) + { + var nugetVersion = new NuGetVersion(version); + + if (!string.IsNullOrWhiteSpace(nugetVersion.Release)) + { + var releaseParts = nugetVersion.Release.Split('-', '.'); + if (releaseParts.Length == 2) + { + if (releaseParts[1].TrimStart('0') == commitCount) + { + // core-sdk does this - OfficialBuildId is only used for their fake package and not in anything shipped + return new DerivedVersion { OfficialBuildId = DateTime.Now.ToString("yyyyMMdd.1"), PreReleaseVersionLabel = releaseParts[0] }; + } + else + { + // NuGet does this - arbitrary build IDs + return new DerivedVersion { OfficialBuildId = releaseParts[1], PreReleaseVersionLabel = releaseParts[0] }; + } + } + else if (releaseParts.Length == 3) + { + // VSTest uses full dates for the first part of their preview build numbers + if (DateTime.TryParseExact(releaseParts[1], "yyyyMMdd", new CultureInfo("en-US"), DateTimeStyles.AssumeLocal, out DateTime fullDate)) + { + return new DerivedVersion { OfficialBuildId = $"{releaseParts[1]}.{releaseParts[2]}", PreReleaseVersionLabel = releaseParts[0] }; + } + else if (int.TryParse(releaseParts[1], out int datePart) && int.TryParse(releaseParts[2], out int buildPart)) + { + if (datePart > 1 && datePart < 8 && buildPart > 1000 && buildPart < 10000) + { + return new DerivedVersion { OfficialBuildId = releaseParts[2], PreReleaseVersionLabel = $"{releaseParts[0]}.{releaseParts[1]}" }; + } + else + { + return new DerivedVersion { OfficialBuildId = $"20{((datePart / 1000))}{((datePart % 1000) / 50):D2}{(datePart % 50):D2}.{buildPart}", PreReleaseVersionLabel = releaseParts[0] }; + } + } + } + else if (releaseParts.Length == 4) + { + // new preview version style, e.g. 5.0.0-preview.7.20365.12 + if (int.TryParse(releaseParts[2], out int datePart) && int.TryParse(releaseParts[3], out int buildPart)) + { + return new DerivedVersion { OfficialBuildId = $"20{((datePart / 1000))}{((datePart % 1000) / 50):D2}{(datePart % 50):D2}.{buildPart}", PreReleaseVersionLabel = $"{releaseParts[0]}.{releaseParts[1]}" }; + } + } + } + else + { + // finalized version number (x.y.z) - probably not our code + // VSTest, Application Insights, Newtonsoft.Json do this + return new DerivedVersion { OfficialBuildId = DateTime.Now.ToString("yyyyMMdd.1"), PreReleaseVersionLabel = string.Empty }; + } + + throw new FormatException($"Can't derive a build ID from version {version} (commit count {commitCount}, release {string.Join(";", nugetVersion.Release.Split('-', '.'))})"); + } + + private static string GetDefaultRepoNameFromUrl(string repoUrl) + { + if (repoUrl.EndsWith(".git")) + { + repoUrl = repoUrl.Substring(0, repoUrl.Length - ".git".Length); + } + return repoUrl.Substring(repoUrl.LastIndexOf("/") + 1); + } + + private static IEnumerable GetRepoNamesOrDefault(Dependency dependency) + { + return dependency.RepoNames ?? new[] { GetDefaultRepoNameFromUrl(dependency.Uri) }; + } + + private static string DeriveRepoGitDirPath(string gitDirPath, string repoUrl) + { + return Path.Combine(gitDirPath, $"{GetDefaultRepoNameFromUrl(repoUrl)}.git"); + } + + private static string DeriveRepoPath(string sourceDirPath, string repoUrl, string hash) + { + // hash could actually be a branch or tag, make it filename-safe + hash = hash.Replace('/', '-').Replace('\\', '-').Replace('?', '-').Replace('*', '-').Replace(':', '-').Replace('|', '-').Replace('"', '-').Replace('<', '-').Replace('>', '-'); + return Path.Combine(sourceDirPath, $"{GetDefaultRepoNameFromUrl(repoUrl)}.{hash}"); + } + + private string GetCommitCount(string gitDir, string hash) + { + return RunGitCommand($"rev-list --count {hash}", gitDir: gitDir); + } + + private string GetCommitDate(string gitDir, string hash) + { + return RunGitCommand($"log -1 --format=%cd --date=short {hash}", gitDir: gitDir); + } + + private IEnumerable GetSubmoduleInfo(string gitModulesFilePath) + { + string submoduleProps = RunGitCommand($"config --file={gitModulesFilePath} --list"); + var submodulePathRegex = new Regex(@"submodule\.(?.*)\.path=(?.*)"); + foreach (Match m in submodulePathRegex.Matches(submoduleProps)) + { + yield return new SubmoduleInfo { Name = m.Groups["submoduleName"].Value, Path = m.Groups["submodulePath"].Value }; + } + } + + private string RunGitCommand(string command, string workTree = null, string gitDir = null) + { + // Windows Git requires these to be before the command + if (workTree != null) + { + command = $"--work-tree={workTree} {command}"; + } + if (gitDir != null) + { + command = $"--git-dir={gitDir} {command}"; + } + + var exec = new Exec + { + BuildEngine = BuildEngine, + Command = $"git {command}", + LogStandardErrorAsError = true, + ConsoleToMSBuild = true, + }; + + if (!exec.Execute() || exec.ExitCode != 0) + { + string error = string.Join(Environment.NewLine, exec.ConsoleOutput.Select(o => o.ItemSpec)); + throw new InvalidOperationException($"git command '{command}' failed with exit code {exec.ExitCode} and error {error ?? ""}"); + } + string output = string.Join(Environment.NewLine, exec.ConsoleOutput.Select(o => o.ItemSpec)); + return output.Trim(); + } + + private void HandleSubmodules(string sourceDirPath, string gitDirPath, Dependency dependency) + { + var gitModulesPath = Path.Combine(sourceDirPath, ".gitmodules"); + foreach (SubmoduleInfo submodule in GetSubmoduleInfo(gitModulesPath)) + { + WriteGitCommitMarkerFileForSubmodule(sourceDirPath, gitDirPath, dependency.Sha, submodule.Name, submodule.Path); + } + } + + private void WriteGitCommitMarkerFileForSubmodule(string sourceDirPath, string gitDirPath, string parentRepoSha, string submoduleName, string submodulePath) + { + var submoduleSha = GetSubmoduleCommit(gitDirPath, parentRepoSha, submodulePath); + var headDirectory = Path.Combine(sourceDirPath, submodulePath, ".git"); + var headPath = Path.Combine(headDirectory, "HEAD"); + Directory.CreateDirectory(headDirectory); + File.WriteAllText(headPath, submoduleSha); + } + + private static void WriteMinimalMetadata(string repoPath, string repoUrl, string hash) + { + var fakeGitDirPath = Path.Combine(repoPath, ".git"); + var fakeGitConfigPath = Path.Combine(fakeGitDirPath, "config"); + var fakeGitHeadPath = Path.Combine(fakeGitDirPath, "HEAD"); + + Directory.CreateDirectory(fakeGitDirPath); + File.WriteAllText(fakeGitHeadPath, hash); + File.WriteAllText(fakeGitConfigPath, $"[remote \"origin\"]{Environment.NewLine}url = \"{repoUrl}\""); + } + + + private string GetSubmoduleCommit(string gitDirPath, string parentRepoSha, string submodulePath) + { + var gitObjectList = RunGitCommand($"ls-tree {parentRepoSha} {submodulePath}", gitDir: gitDirPath); + var submoduleRegex = new Regex(@"\d{6}\s+commit\s+(?[a-fA-F0-9]{40})\s+(.+)"); + var submoduleMatch = submoduleRegex.Match(gitObjectList); + if (!submoduleMatch.Success) + { + throw new InvalidDataException($"Couldn't find a submodule commit in {gitObjectList} for {submodulePath}"); + } + return submoduleMatch.Groups["submoduleSha"].Value; + } + + private static void WritePropsFile(string filePath, Dictionary properties) + { + var content = new StringBuilder(); + content.AppendLine(""); + content.AppendLine(""); + content.AppendLine(" "); + foreach (var propName in properties.Keys.OrderBy(k => k)) + { + content.AppendLine($" <{propName}>{properties[propName]}"); + } + content.AppendLine(" "); + content.AppendLine(""); + File.WriteAllText(filePath, content.ToString()); + } + + private class DerivedVersion + { + internal string OfficialBuildId { get; set; } + internal string PreReleaseVersionLabel { get; set; } + } + + private class SubmoduleInfo + { + internal string Name { get; set; } + internal string Path { get; set; } + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteVersionsFile.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteVersionsFile.cs new file mode 100755 index 000000000..e4ba279f7 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteVersionsFile.cs @@ -0,0 +1,45 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Text; +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Packaging; +using NuGet.Packaging.Core; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class WriteVersionsFile : Task + { + [Required] + public ITaskItem[] NugetPackages { get; set; } + + [Required] + public string OutputPath { get; set; } + + public override bool Execute() + { + Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); + + using (Stream outStream = File.Open(OutputPath, FileMode.Create)) + { + using (StreamWriter sw = new StreamWriter(outStream, new UTF8Encoding(false))) + { + foreach (ITaskItem nugetPackage in NugetPackages) + { + using (PackageArchiveReader par = new PackageArchiveReader(nugetPackage.GetMetadata("FullPath"))) + { + PackageIdentity packageIdentity = par.GetIdentity(); + sw.WriteLine($"{packageIdentity.Id} {packageIdentity.Version}"); + } + } + } + } + + return true; + } + } +} diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ZipFileExtractToDirectory.cs b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ZipFileExtractToDirectory.cs new file mode 100644 index 000000000..bec92355c --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/ZipFileExtractToDirectory.cs @@ -0,0 +1,86 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using System; +using System.IO; +using System.IO.Compression; + +namespace Microsoft.DotNet.Build.Tasks +{ + public sealed class ZipFileExtractToDirectory : BuildTask + { + /// + /// The path to the archive to be extracted. + /// + [Required] + public string SourceArchive { get; set; } + + /// + /// The path of the directory to extract into. + /// + [Required] + public string DestinationDirectory { get; set; } + + /// + /// Indicates if the destination directory should be overwritten if it already exists. + /// + public bool OverwriteDestination { get; set; } + + /// + /// File entries to include in the extraction. Entries are relative + /// paths inside the archive. If null or empty, all files are extracted. + /// + public ITaskItem[] Include { get; set; } + + public override bool Execute() + { + try + { + if (Directory.Exists(DestinationDirectory)) + { + if (OverwriteDestination) + { + Log.LogMessage(MessageImportance.Low, $"'{DestinationDirectory}' already exists, trying to delete before unzipping..."); + Directory.Delete(DestinationDirectory, recursive: true); + } + else + { + Log.LogWarning($"'{DestinationDirectory}' already exists. Did you forget to set '{nameof(OverwriteDestination)}' to true?"); + } + } + + Log.LogMessage(MessageImportance.High, "Decompressing '{0}' into '{1}'...", SourceArchive, DestinationDirectory); + Directory.CreateDirectory(Path.GetDirectoryName(DestinationDirectory)); + + using (ZipArchive archive = ZipFile.OpenRead(SourceArchive)) + { + if (Include?.Length > 0) + { + foreach (ITaskItem entryItem in Include) + { + ZipArchiveEntry entry = archive.GetEntry(entryItem.ItemSpec); + string destinationPath = Path.Combine(DestinationDirectory, entryItem.ItemSpec); + + Directory.CreateDirectory(Path.GetDirectoryName(destinationPath)); + entry.ExtractToFile(destinationPath, overwrite: false); + } + } + else + { + archive.ExtractToDirectory(DestinationDirectory); + } + } + } + catch (Exception e) + { + // We have 2 log calls because we want a nice error message but we also want to capture the callstack in the log. + Log.LogError("An exception has occurred while trying to decompress '{0}' into '{1}'.", SourceArchive, DestinationDirectory); + Log.LogErrorFromException(e, /*show stack=*/ true, /*show detail=*/ true, DestinationDirectory); + return false; + } + return true; + } + } +} \ No newline at end of file diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/_._ b/src/SourceBuild/tarball/content/tools-local/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/_._ new file mode 100644 index 000000000..e69de29bb diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/SourceBuild.MSBuildSdkResolver/SourceBuild.MSBuildSdkResolver.csproj b/src/SourceBuild/tarball/content/tools-local/tasks/SourceBuild.MSBuildSdkResolver/SourceBuild.MSBuildSdkResolver.csproj new file mode 100755 index 000000000..f6fa859e9 --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/SourceBuild.MSBuildSdkResolver/SourceBuild.MSBuildSdkResolver.csproj @@ -0,0 +1,20 @@ + + + + netstandard2.0 + $(DotNetSdkResolversDir)$(MSBuildProjectName)\$(MSBuildProjectName).dll + + + + + + + + + + + + + + + diff --git a/src/SourceBuild/tarball/content/tools-local/tasks/SourceBuild.MSBuildSdkResolver/SourceBuiltSdkResolver.cs b/src/SourceBuild/tarball/content/tools-local/tasks/SourceBuild.MSBuildSdkResolver/SourceBuiltSdkResolver.cs new file mode 100644 index 000000000..250c620bf --- /dev/null +++ b/src/SourceBuild/tarball/content/tools-local/tasks/SourceBuild.MSBuildSdkResolver/SourceBuiltSdkResolver.cs @@ -0,0 +1,200 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using System; +using System.Collections; +using System.Collections.Generic; +using System.IO; +using System.Linq; + +namespace Microsoft.DotNet.SourceBuild.Tasks +{ + /// + /// Extends the SDK to handle "SOURCE_BUILD_SDK_*" override environment variables. Each override + /// should provide a set of 3 environment variables: + /// + /// SOURCE_BUILD_SDK_ID_EXAMPLE=Your.Sdk.Example + /// ID of the SDK nuget package to override. + /// + /// SOURCE_BUILD_SDK_DIR_EXAMPLE=/git/repo/bin/extracted/Your.Sdk.Example/ + /// Directory where the sdk/Sdk.props and/or sdk/Sdk.targets files are located. This should be + /// the directory where the override SDK package is extracted. + /// + /// SOURCE_BUILD_SDK_VERSION_EXAMPLE=1.0.0-source-built + /// (Optional) Version of the SDK package to use. This is informational. + /// + public class SourceBuiltSdkResolver : SdkResolver + { + public override string Name => nameof(SourceBuiltSdkResolver); + + public override int Priority => 0; + + public override SdkResult Resolve( + SdkReference sdkReference, + SdkResolverContext resolverContext, + SdkResultFactory factory) + { + string sdkDescription = sdkReference.Name; + if (!string.IsNullOrEmpty(sdkReference.Version)) + { + sdkDescription += $" {sdkReference.Version}"; + } + if (!string.IsNullOrEmpty(sdkReference.MinimumVersion)) + { + sdkDescription += $" (>= {sdkReference.MinimumVersion})"; + } + + SourceBuiltSdkOverride[] overrides = Environment.GetEnvironmentVariables() + .Cast() + .Select(SourceBuiltSdkOverride.Create) + .Where(o => o != null) + .ToArray(); + + void LogMessage(string message) + { + resolverContext.Logger.LogMessage($"[{Name}] {message}", MessageImportance.High); + } + + if (overrides.Any()) + { + string separator = overrides.Length == 1 ? " " : Environment.NewLine; + + LogMessage( + $"Looking for SDK {sdkDescription}. Detected config(s) in env:{separator}" + + string.Join(Environment.NewLine, overrides.Select(o => o.ToString()))); + } + + SourceBuiltSdkOverride[] matches = overrides + .Where(o => sdkReference.Name.Equals(o?.Id, StringComparison.OrdinalIgnoreCase)) + .ToArray(); + + var unresolvableReasons = new List(); + + if (matches.Length != 1) + { + unresolvableReasons.Add( + $"{matches.Length} overrides found for '{sdkReference.Name}'"); + } + else + { + SourceBuiltSdkOverride match = matches[0]; + string[] matchProblems = match.GetValidationErrors().ToArray(); + + if (matchProblems.Any()) + { + unresolvableReasons.Add($"Found match '{match.Group}' with problems:"); + unresolvableReasons.AddRange(matchProblems); + } + else + { + LogMessage($"Overriding {sdkDescription} with '{match.Group}'"); + + return factory.IndicateSuccess(match.SdkDir, match.Version); + } + } + + return factory.IndicateFailure(unresolvableReasons.Select(r => $"[{Name}] {r}")); + } + + /// + /// Takes a directory path (not ending in a separator) and determines if it is the "sdk" + /// directory inside a SDK package with a case-insensitive comparison. + /// + private static bool IsSdkDirectory(string path) + { + return Path.GetFileName(path).Equals("sdk", StringComparison.OrdinalIgnoreCase); + } + + private class SourceBuiltSdkOverride + { + private const string EnvPrefix = "SOURCE_BUILT_SDK_"; + private const string EnvId = EnvPrefix + "ID_"; + private const string EnvVersion = EnvPrefix + "VERSION_"; + private const string EnvDir = EnvPrefix + "DIR_"; + + public static SourceBuiltSdkOverride Create(DictionaryEntry entry) + { + if (entry.Key is string key && key.StartsWith(EnvId)) + { + // E.g. "ARCADE" from "SOURCE_BUILD_SDK_ID_ARCADE=Microsoft.DotNet.Arcade.Sdk". + string group = key.Substring(EnvId.Length); + + if (string.IsNullOrEmpty(group)) + { + return null; + } + + string id = entry.Value as string; + string version = Environment.GetEnvironmentVariable(EnvVersion + group); + string dir = Environment.GetEnvironmentVariable(EnvDir + group); + + if (string.IsNullOrEmpty(version)) + { + version = "1.0.0-source-built"; + } + + string sdkDir = null; + if (!string.IsNullOrEmpty(dir)) + { + sdkDir = Directory.EnumerateDirectories(dir).FirstOrDefault(IsSdkDirectory); + } + + return new SourceBuiltSdkOverride + { + Group = group, + Id = id, + Version = version, + Dir = dir, + SdkDir = sdkDir, + }; + } + return null; + } + + /// + /// Name of the environment variable group, used to associate the multiple env vars. + /// + public string Group { get; set; } + + /// + /// ID of the SDK package. + /// + public string Id { get; set; } + + /// + /// Version of the source-built SDK package to use instead. + /// + public string Version { get; set; } + + /// + /// Directory where the source-built SDK files are found, in extracted form. + /// + public string Dir { get; set; } + + /// + /// Directory where the Sdk.props/Sdk.targets files are found, inside Dir. + /// + public string SdkDir { get; set; } + + public override string ToString() => $"'{Group}' for '{Id}/{Version}' at '{Dir}'"; + + public IEnumerable GetValidationErrors() + { + if (string.IsNullOrEmpty(Id)) + { + yield return $"'{EnvId}{Group}' not specified."; + } + if (string.IsNullOrEmpty(Dir)) + { + yield return $"'{EnvDir}{Group}' not specified."; + } + else if (string.IsNullOrEmpty(SdkDir)) + { + yield return $"Didn't find any 'sdk' directory in SDK package dir '{Dir}'"; + } + } + } + } +} diff --git a/src/redist/targets/packaging/deb/dotnet-debian_config.json b/src/redist/targets/packaging/deb/dotnet-debian_config.json index 5c0c1e6db..321892b8f 100644 --- a/src/redist/targets/packaging/deb/dotnet-debian_config.json +++ b/src/redist/targets/packaging/deb/dotnet-debian_config.json @@ -22,10 +22,10 @@ "architecture":"any" }, - "copyright": "2015 Microsoft", + "copyright": ".NET Foundation and contributors", "license": { "type": "MIT", - "full_text": "Copyright (c) 2015 Microsoft\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE." + "full_text": "The MIT License (MIT)\nCopyright (c) .NET Foundation and Contributors\n\nAll rights reserved.\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE." }, "debian_dependencies":{ diff --git a/src/redist/targets/packaging/rpm/dotnet-config.json b/src/redist/targets/packaging/rpm/dotnet-config.json index e1ba8e1ee..ed3ccbfeb 100644 --- a/src/redist/targets/packaging/rpm/dotnet-config.json +++ b/src/redist/targets/packaging/rpm/dotnet-config.json @@ -16,10 +16,10 @@ "control": { "architecture": "amd64" }, - "copyright": "2015 Microsoft", + "copyright": ".NET Foundation and contributors", "license": { "type": "MIT", - "full_text": "Copyright (c) 2015 Microsoft\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE." + "full_text": "The MIT License (MIT)\nCopyright (c) .NET Foundation and Contributors\n\nAll rights reserved.\n\nPermission is hereby granted, free of charge, to any person obtaining a copy\nof this software and associated documentation files (the \"Software\"), to deal\nin the Software without restriction, including without limitation the rights\nto use, copy, modify, merge, publish, distribute, sublicense, and/or sell\ncopies of the Software, and to permit persons to whom the Software is\nfurnished to do so, subject to the following conditions:\n\nThe above copyright notice and this permission notice shall be included in all\ncopies or substantial portions of the Software.\n\nTHE SOFTWARE IS PROVIDED \"AS IS\", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR\nIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,\nFITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE\n\nAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER\nLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,\nOUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE\nSOFTWARE." }, "rpm_dependencies": [ {