mirrors/dotnet-installer
2
0
Fork 0
Code Issues Pull requests Projects Releases Packages 1 Wiki Activity Actions

Merge in 'release/6.0.1xx' changes

Browse source
This commit is contained in:
dotnet-bot 2022-02-10 22:10:07 +00:00
commit 8d33a4878c
11 changed files with 1699 additions and 51 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
eng/source-build-tarball-build-official.yml
View file

@ -11,4 +11,4 @@ stages:
jobs:
- template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml
parameters:
tarballResourceId: installer-build-resource
installerBuildResourceId: installer-build-resource

2
eng/source-build-tarball-build.yml
View file

@ -11,4 +11,4 @@ stages:
jobs:
- template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml
parameters:
tarballResourceId: installer-build-resource
installerBuildResourceId: installer-build-resource

40
src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml
View file

@ -7,11 +7,12 @@ parameters:
# Dependent jobs that must be completed before this job will run
dependsOn:
# The resource id of the tarball to download and build
tarballResourceId: current
# Resource id of the installer build to retrieve source/product tarball's from
installerBuildResourceId: current
# The following parameters aren't expected to be passed in rather they are used for encapsulation
# -----------------------------------------------------------------------------------------------
buildJobTimeout: 300
centOS7Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-source-build-20210714125450-5d87b80
centOS8Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-8-source-build-20211118190102-9355e7b
debian9Container: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-stretch-20211001171226-047508b
@ -42,12 +43,14 @@ jobs:
_BuildArch: x64
_Container: ${{ parameters.centOS7Container }}
_ExcludeOmniSharpTests: true
_Platform: linux
_RunOnline: true
CentOS7-Offline:
_BootstrapPrep: true
_BuildArch: x64
_Container: ${{ parameters.centOS7Container }}
_ExcludeOmniSharpTests: true
_Platform: linux
_RunOnline: false
${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
CentOS8-Offline:
@ -55,20 +58,23 @@ jobs:
_BuildArch: x64
_Container: ${{ parameters.centOS8Container }}
_ExcludeOmniSharpTests: false
_Platform: linux
_RunOnline: false
Fedora33-Offline:
_BootstrapPrep: false
_BuildArch: x64
_Container: ${{ parameters.fedora33Container }}
_ExcludeOmniSharpTests: false
_Platform: linux
_RunOnline: false
Ubuntu1804-Offline:
_BootstrapPrep: true
_BuildArch: x64
_Container: ${{ parameters.ubuntu1804Container }}
_ExcludeOmniSharpTests: false
_Platform: linux
_RunOnline: false
timeoutInMinutes: 300
timeoutInMinutes: ${{ parameters.buildJobTimeout }}
variables:
- ${{ if ne(variables['System.TeamProject'], 'public') }}:
- group: AzureDevOps-Artifact-Feeds-Pats
@ -79,20 +85,13 @@ jobs:
- checkout: self
clean: true
- ${{ if ne(variables['System.TeamProject'], 'public') }}:
- task: Bash@3
displayName: Setup Private Feeds Credentials
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
arguments: $(Build.SourcesDirectory)/NuGet.config $Token
env:
Token: $(dn-bot-dnceng-artifact-feeds-rw)
- template: /src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml
parameters:
buildArch: $(_BuildArch)
container: $(_Container)
excludeOmniSharpTests: $(_ExcludeOmniSharpTests)
installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
platform: $(_Platform)
prepScript: |
set -x
@ -104,7 +103,6 @@ jobs:
docker run --rm -v ${{ parameters.tarballDir }}:/tarball -w /tarball $(_Container) ./prep.sh ${customPrepArgs}
runOnline: $(_RunOnline)
tarballDir: ${{ parameters.tarballDir }}
tarballResourceId: ${{ parameters.tarballResourceId }}
- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}:
- job: Rebuild_Tarball
@ -121,8 +119,9 @@ jobs:
_PreviousSourceBuildArtifact: Build Tarball Fedora33-Offline_Artifacts
_BuildArch: x64
_Container: ${{ parameters.fedora33Container }}
_Platform: linux
_RunOnline: false
timeoutInMinutes: 180
timeoutInMinutes: ${{ parameters.buildJobTimeout }}
variables:
- ${{ if ne(variables['System.TeamProject'], 'public') }}:
- group: AzureDevOps-Artifact-Feeds-Pats
@ -133,15 +132,6 @@ jobs:
- checkout: self
clean: true
- ${{ if ne(variables['System.TeamProject'], 'public') }}:
- task: Bash@3
displayName: Setup Private Feeds Credentials
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
arguments: $(Build.SourcesDirectory)/NuGet.config $Token
env:
Token: $(dn-bot-dnceng-artifact-feeds-rw)
- download: current
artifact: $(_PreviousSourceBuildArtifact)
patterns: '*.tar.gz'
@ -159,6 +149,9 @@ jobs:
additionalBuildArgs: --with-sdk /tarball/.dotnet
buildArch: $(_BuildArch)
container: $(_Container)
installerBuildResourceId: ${{ parameters.installerBuildResourceId }}
isBootstrapped: true
platform: $(_Platform)
prepScript: |
set -x
@ -168,4 +161,3 @@ jobs:
eval rm -f "$tarballFilePath"
runOnline: $(_RunOnline)
tarballDir: ${{ parameters.tarballDir }}
tarballResourceId: ${{ parameters.tarballResourceId }}

45
src/SourceBuild/Arcade/eng/common/templates/steps/source-build-build-tarball.yml
View file

@ -13,6 +13,15 @@ parameters:
# Skip running the OmniSharp smoke-tests
excludeOmniSharpTests: false
# Resource id of the installer build to retrieve source/product tarball's from
installerBuildResourceId: null
# Whether or not this is a bootstrapped build
isBootstrapped: false
# Platform the build is running on (linux, osx, win)
platform: null
# Script logic to prep the tarball for building
prepScript: null
@ -22,21 +31,32 @@ parameters:
# Directory to build the tarball in
tarballDir:
# Resource id of the tarball to download and build
tarballResourceId: null
steps:
- download: ${{ parameters.tarballResourceId }}
- ${{ if ne(variables['System.TeamProject'], 'public') }}:
- task: Bash@3
displayName: Setup Private Feeds Credentials
inputs:
filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh
arguments: $(Build.SourcesDirectory)/NuGet.config $Token
env:
Token: $(dn-bot-dnceng-artifact-feeds-rw)
- download: ${{ parameters.installerBuildResourceId }}
artifact: BlobArtifacts
patterns: '**/dotnet-sdk-source*.tar.gz'
displayName: Download Tarball
displayName: Download Source Tarball
- download: ${{ parameters.installerBuildResourceId }}
artifact: BlobArtifacts
patterns: '**/dotnet-sdk-!(*-*)-${{ parameters.platform }}-${{ parameters.buildArch }}.tar.gz'
displayName: Download MSFT sdk Tarball
- script: |
set -x
resourceIdPathSegment=
if [ '${{ parameters.tarballResourceId }}' != 'current' ]; then
resourceIdPathSegment='${{ parameters.tarballResourceId }}/'
if [ '${{ parameters.installerBuildResourceId }}' != 'current' ]; then
resourceIdPathSegment='${{ parameters.installerBuildResourceId }}/'
fi
mkdir -p "${{ parameters.tarballDir }}"
@ -70,7 +90,16 @@ steps:
rm -f ${{ parameters.tarballDir }}/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/online.NuGet.Config
cp NuGet.config ${{ parameters.tarballDir }}/test/Microsoft.DotNet.SourceBuild.SmokeTests/smoke-tests/online.NuGet.Config
docker run --rm -v ${{ parameters.tarballDir }}:/tarball -w /tarball -e EXCLUDE_OMNISHARP_TESTS=${{ parameters.excludeOmniSharpTests}} ${{ parameters.container }} ./build.sh --run-smoke-test ${{ parameters.additionalBuildArgs }}
dockerVolumeArgs="-v ${{ parameters.tarballDir }}:/tarball"
dockerEnvArgs="-e EXCLUDE_OMNISHARP_TESTS=${{ parameters.excludeOmniSharpTests}}"
if [ '${{ parameters.isBootstrapped}}' != 'true' && '${{ parameters.installerBuildResourceId }}' != 'current' ]; then
dockerVolumeArgs+=" -v $(PIPELINE.WORKSPACE)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/:/BlobArtifacts"
msftSdkTarballName=$(find "$(PIPELINE.WORKSPACE)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/" -name "dotnet-sdk-*-${{ parameters.Platform }}-${{ parameters.buildArch }}.tar.gz" -exec basename {} \;)
dockerEnvArgs+=" -e MSFT_SDK_TARBALL_PATH=/BlobArtifacts/$msftSdkTarballName"
fi
docker run --rm $dockerVolumeArgs -w /tarball $dockerEnvArgs ${{ parameters.container }} ./build.sh --run-smoke-test ${{ parameters.additionalBuildArgs }}
displayName: Run Tests
# Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph.

31
src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/BaselineHelper.cs
View file

@ -4,9 +4,11 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using Xunit;
using Xunit.Abstractions;
namespace Microsoft.DotNet.SourceBuild.SmokeTests
{
@ -32,6 +34,35 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests
Assert.Null(message);
}
public static void Compare(string baselineFileName, string actual, ITestOutputHelper outputHelper)
{
string baselineFilePath = GetBaselineFilePath(baselineFileName);
string baseline = File.ReadAllText(baselineFilePath);
string? message = null;
if (baseline != actual)
{
string actualBaselineFilePath = Path.Combine(Environment.CurrentDirectory, $"{baselineFileName}");
File.WriteAllText(actualBaselineFilePath, actual);
// Retrieve a diff in order to provide a UX which calls out the diffs.
string diff = DiffFiles(baselineFilePath, actualBaselineFilePath, outputHelper);
message = $"{Environment.NewLine}Baseline '{baselineFilePath}' does not match actual '{actualBaselineFilePath}`. {Environment.NewLine}"
+ $"{diff}{Environment.NewLine}";
}
Assert.Null(message);
}
public static string DiffFiles(string file1Path, string file2Path, ITestOutputHelper outputHelper)
{
(Process Process, string StdOut, string StdErr) diffResult =
ExecuteHelper.ExecuteProcess("git", $"diff --no-index {file1Path} {file2Path}", outputHelper);
Assert.Equal(1, diffResult.Process.ExitCode);
return diffResult.StdOut;
}
private static string GetBaselineFilePath(string baselineFileName) => Path.Combine(Directory.GetCurrentDirectory(), "baselines", baselineFileName);
}
}

6
src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/Config.cs
View file

@ -3,17 +3,21 @@
// See the LICENSE file in the project root for more information.
using System;
using System.IO;
namespace Microsoft.DotNet.SourceBuild.SmokeTests;
internal static class Config
{
public static string DotNetDirectory { get; } = Environment.GetEnvironmentVariable("DOTNET_DIR") ?? "./.dotnet";
public static string DotNetDirectory { get; } =
Environment.GetEnvironmentVariable("DOTNET_DIR") ?? Path.Combine(Directory.GetCurrentDirectory(), ".dotnet");
public static string DotNetTarballPath { get; } = Environment.GetEnvironmentVariable(DotNetTarballPathEnv) ?? string.Empty;
public const string DotNetTarballPathEnv = "DOTNET_TARBALL_PATH";
public static bool ExcludeOmniSharpTests { get; } =
bool.TryParse(Environment.GetEnvironmentVariable("EXCLUDE_OMNISHARP_TESTS"), out bool excludeOmniSharpTests) ? excludeOmniSharpTests : false;
public static bool ExcludeOnlineTests { get; } =
bool.TryParse(Environment.GetEnvironmentVariable("EXCLUDE_ONLINE_TESTS"), out bool excludeOnlineTests) ? excludeOnlineTests : false;
public static string MsftSdkTarballPath { get; } = Environment.GetEnvironmentVariable(MsftSdkTarballPathEnv) ?? string.Empty;
public const string MsftSdkTarballPathEnv = "MSFT_SDK_TARBALL_PATH";
public static string TargetRid { get; } = Environment.GetEnvironmentVariable("TARGET_RID") ?? string.Empty;
}

21
src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/DotNetHelper.cs
View file

@ -12,24 +12,27 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests;
internal class DotNetHelper
{
private static readonly object s_lockObj = new object();
public string DotNetPath { get; }
public string DotNetInstallDirectory { get; }
public DotNetHelper(ITestOutputHelper outputHelper)
{
if (!Directory.Exists(Config.DotNetDirectory))
lock (s_lockObj)
{
if (!File.Exists(Config.DotNetTarballPath))
if (!Directory.Exists(Config.DotNetDirectory))
{
throw new InvalidOperationException($"Tarball path '{Config.DotNetTarballPath}' specified in {Config.DotNetTarballPathEnv} does not exist.");
}
if (!File.Exists(Config.DotNetTarballPath))
{
throw new InvalidOperationException($"Tarball path '{Config.DotNetTarballPath}' specified in {Config.DotNetTarballPathEnv} does not exist.");
}
Directory.CreateDirectory(Config.DotNetDirectory);
ExecuteHelper.ExecuteProcess("tar", $"xzf {Config.DotNetTarballPath} -C {Config.DotNetDirectory}", outputHelper);
Directory.CreateDirectory(Config.DotNetDirectory);
ExecuteHelper.ExecuteProcessValidateExitCode("tar", $"xzf {Config.DotNetTarballPath} -C {Config.DotNetDirectory}", outputHelper);
}
}
DotNetInstallDirectory = Path.Combine(Directory.GetCurrentDirectory(), Config.DotNetDirectory);
DotNetPath = Path.Combine(DotNetInstallDirectory, "dotnet");
DotNetPath = Path.Combine(Config.DotNetDirectory, "dotnet");
}
public void ExecuteDotNetCmd(string args, ITestOutputHelper outputHelper)

23
src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/ExecuteHelper.cs
View file

@ -2,6 +2,7 @@
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Diagnostics;
using System.Linq;
using System.Text;
@ -12,7 +13,7 @@ namespace Microsoft.DotNet.SourceBuild.SmokeTests;
internal static class ExecuteHelper
{
public static (Process Process, string StdOut, string StdErr) ExecuteProcess(
string fileName, string args, ITestOutputHelper outputHelper)
string fileName, string args, ITestOutputHelper outputHelper, bool logOutput = false)
{
outputHelper.WriteLine($"Executing: {fileName} {args}");
@ -47,17 +48,33 @@ internal static class ExecuteHelper
process.WaitForExit();
string output = stdOutput.ToString().Trim();
if (outputHelper != null && !string.IsNullOrWhiteSpace(output))
if (logOutput && !string.IsNullOrWhiteSpace(output))
{
outputHelper.WriteLine(output);
}
string error = stdError.ToString().Trim();
if (outputHelper != null && !string.IsNullOrWhiteSpace(error))
if (logOutput && !string.IsNullOrWhiteSpace(error))
{
outputHelper.WriteLine(error);
}
return (process, output, error);
}
public static string ExecuteProcessValidateExitCode(string fileName, string args, ITestOutputHelper outputHelper)
{
(Process Process, string StdOut, string StdErr) result = ExecuteHelper.ExecuteProcess(fileName, args, outputHelper);
if (result.Process.ExitCode != 0)
{
ProcessStartInfo startInfo = result.Process.StartInfo;
string msg = $"Failed to execute {startInfo.FileName} {startInfo.Arguments}" +
$"{Environment.NewLine}Exit code: {result.Process.ExitCode}" +
$"{Environment.NewLine}Standard Error: {result.StdErr}";
throw new InvalidOperationException(msg);
}
return result.StdOut;
}
}

92
src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/SdkContentTests.cs Normal file
View file

@ -0,0 +1,92 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.
// See the LICENSE file in the project root for more information.
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
using Xunit;
using Xunit.Abstractions;
namespace Microsoft.DotNet.SourceBuild.SmokeTests;
public class SdkContentTests
{
private ITestOutputHelper OutputHelper { get; }
private DotNetHelper DotNetHelper { get; }
public SdkContentTests(ITestOutputHelper outputHelper)
{
OutputHelper = outputHelper;
DotNetHelper = new DotNetHelper(outputHelper);
}
/// <Summary>
/// Verifies the file layout of the source built sdk tarball to the Microsoft build.
/// The differences are captured in baselines/MsftToSbSdkDiff.txt.
/// Version numbers that appear in paths are compared but are stripped from the baseline.
/// This makes the baseline durable between releases. This does mean however, entries
/// in the baseline may appear identical if the diff is version specific.
/// </Summary>
[Fact]
public void CompareMsftToSb()
{
if (string.IsNullOrWhiteSpace(Config.MsftSdkTarballPath))
{
OutputHelper.WriteLine($"skipping {nameof(CompareMsftToSb)} because {Config.MsftSdkTarballPathEnv} was not specified.");
return;
}
if (!File.Exists(Config.MsftSdkTarballPath))
{
throw new InvalidOperationException($"Tarball path '{Config.MsftSdkTarballPath}' specified in {Config.MsftSdkTarballPathEnv} does not exist.");
}
const string msftFileListingFileName = "msftSdkFiles.txt";
const string sbFileListingFileName = "sbSdkFiles.txt";
WriteTarballFileList(Config.MsftSdkTarballPath, msftFileListingFileName);
WriteTarballFileList(Config.DotNetTarballPath, sbFileListingFileName);
string diff = BaselineHelper.DiffFiles(msftFileListingFileName, sbFileListingFileName, OutputHelper);
diff = RemoveVersionedPaths(diff);
diff = RemoveDiffMarkers(diff);
diff = RemoveRids(diff);
BaselineHelper.Compare("MsftToSbSdk.diff", diff, OutputHelper);
}
private void WriteTarballFileList(string tarballPath, string outputFileName)
{
string fileListing = ExecuteHelper.ExecuteProcessValidateExitCode("tar", $"tf {tarballPath}", OutputHelper);
IEnumerable<string> files = fileListing.Split(Environment.NewLine).OrderBy(path => path);
File.WriteAllLines(outputFileName, files);
}
private static string RemoveDiffMarkers(string source)
{
Regex indexRegex = new("^index .*", RegexOptions.Multiline);
string result = indexRegex.Replace(source, "index ------------");
Regex diffSegmentRegex = new("^@@ .* @@", RegexOptions.Multiline);
return diffSegmentRegex.Replace(result, "@@ ------------ @@");
}
private string RemoveRids(string diff) => diff.Replace(Config.TargetRid, "bannana.rid");
private static string RemoveVersionedPaths(string source)
{
// Remove semantic version path segments
string pathSeparator = Regex.Escape(Path.DirectorySeparatorChar.ToString());
// Regex source: https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string
Regex semanticVersionRegex = new(
$"{pathSeparator}(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)"
+ $"(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))"
+ $"?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?{pathSeparator}");
string result = semanticVersionRegex.Replace(source, $"{Path.DirectorySeparatorChar}x.y.z{Path.DirectorySeparatorChar}");
// Remove net.x.y path segments
Regex netTfmRegex = new($"{pathSeparator}net[1-9]*.[0-9]{pathSeparator}");
return netTfmRegex.Replace(result, $"{Path.DirectorySeparatorChar}netx.y{Path.DirectorySeparatorChar}");
}
}

7
src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/XmlDocTests.cs
View file

@ -6,7 +6,6 @@ using System.Collections.Generic;
using System.IO;
using Xunit;
using Xunit.Abstractions;
using System;
using System.Linq;
namespace Microsoft.DotNet.SourceBuild.SmokeTests;
@ -31,7 +30,7 @@ public class XmlDocTests
{
List<string> missingXmlDoc = new();
string targetingPacksDirectory = Path.Combine(DotNetHelper.DotNetInstallDirectory, "packs");
string targetingPacksDirectory = Path.Combine(Config.DotNetDirectory, "packs");
foreach (string targetingPackAssembly in Directory.EnumerateFiles(targetingPacksDirectory, "*.dll", SearchOption.AllDirectories))
{
if (targetingPackAssembly.EndsWith("resources.dll"))
@ -42,8 +41,8 @@ public class XmlDocTests
string xmlFile = Path.ChangeExtension(targetingPackAssembly, ".xml");
if (!File.Exists(xmlFile))
{
string pathWithoutPacksPrefix = xmlFile.Substring(targetingPacksDirectory.Length + 1);
String[] pathParts = pathWithoutPacksPrefix.Split(Path.DirectorySeparatorChar);
string pathWithoutPacksPrefix = xmlFile[(targetingPacksDirectory.Length + 1)..];
string[] pathParts = pathWithoutPacksPrefix.Split(Path.DirectorySeparatorChar);
string pathWithoutVersion = string.Join(Path.DirectorySeparatorChar, pathParts.Take(1).Concat(pathParts.Skip(2)));
missingXmlDoc.Add(pathWithoutVersion);
}

1481
src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/baselines/MsftToSbSdk.diff Normal file
View file

File diff suppressed because it is too large Load diff