From e7e39f547242bb4c19ea1ac2e88bf62b0695f48f Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 29 Nov 2022 13:28:55 +0000 Subject: [PATCH 1/9] Update dependencies from https://github.com/dotnet/arcade build 20221128.1 Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.CMake.Sdk From Version 8.0.0-beta.22575.1 -> To Version 8.0.0-beta.22578.1 --- eng/Version.Details.xml | 12 ++++++------ eng/Versions.props | 2 +- global.json | 4 ++-- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 6214ce3e5..cd4bdbfe2 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -192,18 +192,18 @@ - + https://github.com/dotnet/arcade - b61fcdfe86b30fe91898bfb296f4f4396fe5d87a + 3817f2b38a05c37c950715f84c95d452bcacfe52 - + https://github.com/dotnet/arcade - b61fcdfe86b30fe91898bfb296f4f4396fe5d87a + 3817f2b38a05c37c950715f84c95d452bcacfe52 - + https://github.com/dotnet/arcade - b61fcdfe86b30fe91898bfb296f4f4396fe5d87a + 3817f2b38a05c37c950715f84c95d452bcacfe52 https://github.com/dotnet/arcade-services diff --git a/eng/Versions.props b/eng/Versions.props index d409f73a0..24382fc64 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -26,7 +26,7 @@ - 8.0.0-beta.22575.1 + 8.0.0-beta.22578.1 diff --git a/global.json b/global.json index c600b9564..10daad3f5 100644 --- a/global.json +++ b/global.json @@ -11,7 +11,7 @@ "cmake": "3.21.0" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.22575.1", - "Microsoft.DotNet.CMake.Sdk": "8.0.0-beta.22575.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.22578.1", + "Microsoft.DotNet.CMake.Sdk": "8.0.0-beta.22578.1" } } From 845f82f047e608a4db44d725851fa1bcefe49276 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Tue, 29 Nov 2022 21:42:01 +0000 Subject: [PATCH 2/9] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20221129.1 Microsoft.SourceBuild.Intermediate.source-build-reference-packages From Version 8.0.0-alpha.1.22578.1 -> To Version 8.0.0-alpha.1.22579.1 --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index d15b57e28..bffbb966e 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -217,9 +217,9 @@ https://github.com/dotnet/runtime af841c8b33cecc92d74222298f1e45bf7bf3d90a - + https://github.com/dotnet/source-build-reference-packages - 7ec4b0aabc55efa11515c987fb0251d39c29f06c + bed0fd20a5a55843c1f04ef195ef2fd7171ae715 From 2adbf40116866ca7f85e69a7ed5406ef4e8ff986 Mon Sep 17 00:00:00 2001 From: Dan Moseley Date: Tue, 29 Nov 2022 15:31:11 -0700 Subject: [PATCH 3/9] Use ".NET" in preference to ".NET Core" (#14902) * Use ".NET" in preference to ".NET Core" We're consistently using ".NET" for recent downloads (eg see https://dotnet.microsoft.com/en-us/download) and increasingly it is clearer to use ".NET" instead of ".NET Core". Change the wording, and explain the meanings. * Update README.md * Apply suggestions from code review Co-authored-by: Maira Wenzel * feedbacK * Apply suggestions from code review Co-authored-by: Maira Wenzel Co-authored-by: Maira Wenzel --- README.md | 27 ++++++++++++++++++--------- 1 file changed, 18 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 811a07c0d..4a1955267 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -# .NET Core SDK +# .NET SDK Installers [![Join the chat at https://gitter.im/dotnet/cli](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/dotnet/cli?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [![GitHub release](https://img.shields.io/github/release/dotnet/installer.svg)](https://GitHub.com/dotnet/installer/releases/) @@ -11,13 +11,22 @@ [![GitHub contributors](https://img.shields.io/github/contributors/dotnet/installer.svg)](https://GitHub.com/dotnet/installer/graphs/contributors/) [![Commit Activity](https://img.shields.io/github/commit-activity/m/dotnet/installer)]() +This repo contains the source code for the cross-platform [.NET](http://github.com/dotnet/core) SDK. It aggregates the .NET toolchain, the .NET runtime, the templates, and the .NET Windows Desktop runtime. It produces zip, tarballs, and native packages for various supported platforms. -This repo contains the source code for the cross-platform [.NET Core](http://github.com/dotnet/core) SDK. It aggregates the .NET Toolchain, the .NET Core runtime, the templates, and the .NET Core Windows Desktop runtime. It produces zip, tarballs, and native packages for various supported platforms. - -Looking for released versions of the .NET Core tooling? +Looking for released versions of the .NET tooling? ---------------------------------------- -Download released versions of the .NET Core tools (CLI, MSBuild and the new csproj) at https://dot.net/core. +The links below are for preview versions of .NET tooling. Prefer to use released versions of the .NET tools? Go to https://dot.net/download. + +Looking for .NET Framework downloads? +---------------------------------------- + +.NET Framework is the product from which the .NET Core project originated. .NET Core (mostly just called ".NET" here) adds many features and improvements and supports many more platforms than .NET Framework. .NET Framework remains fully supported and you can find the downloads on the [.NET website](https://dotnet.microsoft.com/download/dotnet-framework). For new projects, we recommend you use .NET Core. + +Want to contribute or find out more about the .NET project? +---------------------------------------- + +This repo is for the installers. Most of the implementation is in other repos, such as the [dotnet/runtime repo](https://github.com/dotnet/runtime) or the [dotnet/aspnetcore repo](https://github.com/dotnet/aspnetcore) and [many others](https://github.com/dotnet/core/blob/main/Documentation/core-repos.md). We welcome you to join us there! Found an issue? --------------- @@ -25,7 +34,7 @@ You can consult the [Documents Index for the SDK repo](https://github.com/dotnet This project has adopted the code of conduct defined by the [Contributor Covenant](http://contributor-covenant.org/) to clarify expected behavior in our community. For more information, see the [.NET Foundation Code of Conduct](http://www.dotnetfoundation.org/code-of-conduct). -# Build .NET installer +# Build .NET installer The repository contains native code project required for the Windows installer. If you intend to build it locally on Windows, you will need to ensure that you have the following items installed. - Install CMAKE 3.21.0 is required if you're building VS 17.0. Make sure to add CMAKE to your PATH (the installer will prompt you). @@ -109,7 +118,7 @@ Visibility|All legs| ## Installers and Binaries -You can download the .NET Core SDK as either an installer (MSI, PKG) or a zip (zip, tar.gz). The .NET Core SDK contains both the .NET Core runtime and CLI tools. +You can download the .NET SDK as either an installer (MSI, PKG) or a zip (zip, tar.gz). The .NET SDK contains both the .NET runtime and CLI tools. **Note:** Be aware that the following installers are the **latest bits**. If you want to install the latest released versions, check out the [preceding section](#looking-for-released-versions-of-the-net-core-tooling). @@ -135,7 +144,7 @@ With development builds, internal NuGet feeds are necessary for some scenarios ( ``` -Please do not directly edit the table below. Use https://github.com/dotnet/installer/tree/main/tools/sdk-readme-table-generator to help you generate it. Make sure to run the table generator test and make any changes to the generator along with your changes to the table. Daily servicing builds have been removed as all servicing is done in private repos to avoid disclosure of critical security fixes. All public servicing builds can be downloaded at http://aka.ms/dotnet-download. +Do not directly edit the table below. Use https://github.com/dotnet/installer/tree/main/tools/sdk-readme-table-generator to help you generate it. Make sure to run the table generator test and make any changes to the generator along with your changes to the table. Daily servicing builds aren't shown here because they may contain upcoming security fixes. All public servicing builds can be downloaded at http://aka.ms/dotnet-download. ### Table *Note* the 7.0.100 build will be finished internally. Below is the last public version available from that branch but is not fully updated with the final runtime. @@ -161,7 +170,7 @@ Reference notes: > * [Host, Host FX Resolver, and Shared Framework](https://github.com/dotnet/runtime/blob/main/docs/project/dogfooding.md#nightly-builds-table) > * [ASP.NET Core Shared Framework](https://github.com/aspnet/AspNetCore/blob/main/docs/DailyBuilds.md) -.NET Core SDK 2.x downloads can be found here: [.NET Core SDK 2.x Installers and Binaries](Downloads2.x.md) +.NET Core SDK 2.x downloads can be found at [.NET Core SDK 2.x Installers and Binaries](Downloads2.x.md) but they are [out of support](https://dotnet.microsoft.com/platform/support/policy/dotnet-core). [win-x64-badge-main]: https://aka.ms/dotnet/8.0.1xx/daily/win_x64_Release_version_badge.svg [win-x64-version-main]: https://aka.ms/dotnet/8.0.1xx/daily/productCommit-win-x64.txt From 2a59f5d06822542d75cae3e3ba32322ad6a0e1bb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Wed, 30 Nov 2022 16:53:15 +0100 Subject: [PATCH 4/9] Add source-build patches for nuget-client's NuGet configs (#15061) --- ...ve-non-compliant-feeds-NuGet.configs.patch | 88 +++++++++++++++++++ 1 file changed, 88 insertions(+) create mode 100644 src/SourceBuild/tarball/patches/nuget-client/0001-Remove-non-compliant-feeds-NuGet.configs.patch diff --git a/src/SourceBuild/tarball/patches/nuget-client/0001-Remove-non-compliant-feeds-NuGet.configs.patch b/src/SourceBuild/tarball/patches/nuget-client/0001-Remove-non-compliant-feeds-NuGet.configs.patch new file mode 100644 index 000000000..f225d9a60 --- /dev/null +++ b/src/SourceBuild/tarball/patches/nuget-client/0001-Remove-non-compliant-feeds-NuGet.configs.patch @@ -0,0 +1,88 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Premek Vysoky +Date: Wed, 30 Nov 2022 11:08:09 +0100 +Subject: [PATCH] Remove non-compliant feeds/NuGet.configs + +--- + NuGet.Config | 14 +++++++------- + submodules/Common/NuGet.config | 7 ------- + submodules/FileSystem/NuGet.config | 7 ------- + 3 files changed, 7 insertions(+), 21 deletions(-) + delete mode 100644 submodules/Common/NuGet.config + delete mode 100644 submodules/FileSystem/NuGet.config + +diff --git a/NuGet.Config b/NuGet.Config +index 0a4c87ea06..7b913598f6 100644 +--- a/NuGet.Config ++++ b/NuGet.Config +@@ -4,9 +4,9 @@ + + + +- + +- ++ ++ + + + +@@ -43,16 +43,13 @@ + + + +- ++ + + + + + +- +- +- +- ++ + + + +@@ -68,6 +65,9 @@ + + + ++ ++ ++ + + + +diff --git a/submodules/Common/NuGet.config b/submodules/Common/NuGet.config +deleted file mode 100644 +index 09c3488420..0000000000 +--- a/submodules/Common/NuGet.config ++++ /dev/null +@@ -1,7 +0,0 @@ +- +- +- +- +- +- +- +diff --git a/submodules/FileSystem/NuGet.config b/submodules/FileSystem/NuGet.config +deleted file mode 100644 +index 1707938c61..0000000000 +--- a/submodules/FileSystem/NuGet.config ++++ /dev/null +@@ -1,7 +0,0 @@ +- +- +- +- +- +- +- +\ No newline at end of file +-- +2.30.2.windows.1 + From b4f77ecac22701bf0aa17e43d4d3c3430c743efb Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 30 Nov 2022 08:23:43 -0800 Subject: [PATCH 5/9] Update dependencies from https://github.com/dotnet/source-build-externals build 20221129.1 (#15063) Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.22578.2 -> To Version 8.0.0-alpha.1.22579.1 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 1214abda4..a686a4e4f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -180,9 +180,9 @@ c3ad00ae84489071080a606f6a8e43c9a91a5cc2 - + https://github.com/dotnet/source-build-externals - af8eedc4daa3d1e29af9d493f657b1927b24c957 + dd9d921db96b4469ebac73ee708cce8a31845c0c From ecb14cfee1acef34a1efef7357ee36169d2a6134 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 30 Nov 2022 08:23:50 -0800 Subject: [PATCH 6/9] Update dependencies from https://github.com/dotnet/arcade build 20221129.2 (#15062) Microsoft.DotNet.Arcade.Sdk , Microsoft.DotNet.Build.Tasks.Installers , Microsoft.DotNet.CMake.Sdk From Version 8.0.0-beta.22578.1 -> To Version 8.0.0-beta.22579.2 Dependency coherency updates Microsoft.DotNet.XliffTasks From Version 1.0.0-beta.22555.1 -> To Version 1.0.0-beta.22576.1 (parent: Microsoft.DotNet.Arcade.Sdk Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 16 ++++++++-------- eng/Versions.props | 2 +- eng/common/tools.sh | 2 +- global.json | 4 ++-- 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index a686a4e4f..cf024ec69 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -192,18 +192,18 @@ - + https://github.com/dotnet/arcade - 3817f2b38a05c37c950715f84c95d452bcacfe52 + 6b3bad6673f3ebe89ebe12ea7c4eff1705b893e6 - + https://github.com/dotnet/arcade - 3817f2b38a05c37c950715f84c95d452bcacfe52 + 6b3bad6673f3ebe89ebe12ea7c4eff1705b893e6 - + https://github.com/dotnet/arcade - 3817f2b38a05c37c950715f84c95d452bcacfe52 + 6b3bad6673f3ebe89ebe12ea7c4eff1705b893e6 https://github.com/dotnet/arcade-services @@ -227,9 +227,9 @@ d047202874ad79d72c75b6354c0f8a9a12d1b054 - + https://github.com/dotnet/xliff-tasks - 52e3661e093095cf353a4531cbfac316e487c4c0 + 9d44d854356c5ea0152b40acdef20c23bde58b03 diff --git a/eng/Versions.props b/eng/Versions.props index 24382fc64..f50bc263f 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -26,7 +26,7 @@ - 8.0.0-beta.22578.1 + 8.0.0-beta.22579.2 diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 6586eab45..e7d1e3897 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -521,7 +521,7 @@ global_json_file="${repo_root}global.json" # determine if global.json contains a "runtimes" entry global_json_has_runtimes=false if command -v jq &> /dev/null; then - if jq -er '. | select(has("runtimes"))' "$global_json_file" &> /dev/null; then + if jq -e '.tools | has("runtimes")' "$global_json_file" &> /dev/null; then global_json_has_runtimes=true fi elif [[ "$(cat "$global_json_file")" =~ \"runtimes\"[[:space:]\:]*\{ ]]; then diff --git a/global.json b/global.json index 10daad3f5..e8fbb1a59 100644 --- a/global.json +++ b/global.json @@ -11,7 +11,7 @@ "cmake": "3.21.0" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.22578.1", - "Microsoft.DotNet.CMake.Sdk": "8.0.0-beta.22578.1" + "Microsoft.DotNet.Arcade.Sdk": "8.0.0-beta.22579.2", + "Microsoft.DotNet.CMake.Sdk": "8.0.0-beta.22579.2" } } From 0f7d074e0f7254a39cb5c99c9cc47cbf5c366ff4 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Wed, 30 Nov 2022 10:41:54 -0800 Subject: [PATCH 7/9] Update dependencies from https://github.com/dotnet/source-build-reference-packages build 20221130.1 (#15065) Microsoft.SourceBuild.Intermediate.source-build-reference-packages From Version 8.0.0-alpha.1.22579.1 -> To Version 8.0.0-alpha.1.22580.1 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index cf024ec69..21b90cf27 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -217,9 +217,9 @@ https://github.com/dotnet/runtime af841c8b33cecc92d74222298f1e45bf7bf3d90a - + https://github.com/dotnet/source-build-reference-packages - bed0fd20a5a55843c1f04ef195ef2fd7171ae715 + 4ed7440e89d5fe7d4375102a441c713fadd5357c From 87955496af2d49999de3f51e6633131bc19b6786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C5=99emek=20Vysok=C3=BD?= Date: Thu, 1 Dec 2022 10:51:39 +0100 Subject: [PATCH 8/9] Use VMR instead of the tarball for Source-Build builds (#15042) For PRs, the Source-Build leg that was running inside of the Build stage is now moved to a separate stage but runs more or less the same: https://dev.azure.com/dnceng-public/public/_build/results?buildId=97509&view=results Instead of creating the tarball, we are building the `dotnet/dotnet` repo there. For internal rolling builds, we are taking [this pipeline](https://dev.azure.com/dnceng/internal/_build/results?buildId=2056327&view=results) and merging it into `dotnet-installer-official-ci`. So it's one extra stage that runs pretty quick (faster than the Build stage by far). It won't be creating and pushing the tarball artifact anymore though. Once the rolling build is finished, there won't be no more source-build-build pipeline but instead dotnet-dotnet-official-ci which will build the dotnet/dotnet repo again instead of the tarball that was originally produced from the rolling build. The MSFT SDK from the installer build will still be consumed by it though. More details https://github.com/dotnet/arcade/issues/10677 --- .vsts-ci.yml | 35 ++- eng/pipelines/templates/jobs/vmr-build.yml | 225 ++++++++++++++++++ .../jobs}/vmr-synchronization.yml | 57 +++-- eng/pipelines/templates/stages/vmr-build.yml | 167 +++++++++++++ eng/pipelines/templates/steps/vmr-prepare.yml | 35 ++- .../templates/steps/vmr-pull-updates.yml | 27 +-- .../templates/variables/vmr-variables.yml | 9 - eng/pipelines/vmr-ci.yml | 26 ++ eng/pipelines/vmr-validation.yml | 51 ---- eng/source-build-tarball-build-official.yml | 26 -- eng/source-build-tarball-build.yml | 14 -- .../job/source-build-build-tarball.yml | 205 ---------------- .../job/source-build-create-tarball.yml | 85 ------- .../job/source-build-run-tarball-build.yml | 121 ---------- 14 files changed, 515 insertions(+), 568 deletions(-) create mode 100644 eng/pipelines/templates/jobs/vmr-build.yml rename eng/pipelines/{ => templates/jobs}/vmr-synchronization.yml (52%) create mode 100644 eng/pipelines/templates/stages/vmr-build.yml delete mode 100644 eng/pipelines/templates/variables/vmr-variables.yml create mode 100644 eng/pipelines/vmr-ci.yml delete mode 100644 eng/pipelines/vmr-validation.yml delete mode 100644 eng/source-build-tarball-build-official.yml delete mode 100644 eng/source-build-tarball-build.yml delete mode 100644 src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml delete mode 100644 src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml delete mode 100644 src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml diff --git a/.vsts-ci.yml b/.vsts-ci.yml index 78fc837c2..1e0355974 100644 --- a/.vsts-ci.yml +++ b/.vsts-ci.yml @@ -10,6 +10,19 @@ trigger: - internal/release/6.* - internal/release/7.* +resources: + repositories: + - repository: vmr + type: github + name: dotnet/dotnet + endpoint: dotnet + +parameters: +- name: vmrBranch + displayName: dotnet/dotnet branch to push to + type: string + default: $(Build.SourceBranchName) + variables: - name: _PublishUsingPipelines value: false @@ -345,12 +358,24 @@ stages: - template: /eng/common/templates/jobs/source-build.yml - - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml - - - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml +- ${{ if and(eq(variables['System.TeamProject'], 'internal'), ne(variables['Build.Reason'], 'PullRequest')) }}: + - stage: Synchronize_VMR + displayName: Synchronize VMR + dependsOn: [] + jobs: + - template: /eng/pipelines/templates/jobs/vmr-synchronization.yml parameters: - dependsOn: Source_Build_Create_Tarball + vmrBranch: ${{ parameters.vmrBranch }} + +# In case the VMR Build stage needs to be temporarily disabled, please make sure that +# the VMR synchronization step is run to validate that the PR can be merged and later +# synchronized into the VMR without problems. +# You can do this by allowing the Synchronize_VMR stage from above in PRs. +- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - template: /eng/pipelines/templates/stages/vmr-build.yml + parameters: + vmrBranch: ${{ parameters.vmrBranch }} + installerBuildResourceId: current - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - stage: Publish diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml new file mode 100644 index 000000000..d7337f38e --- /dev/null +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -0,0 +1,225 @@ +### This job source-builds https://github.com/dotnet/dotnet with given parameters +### If run in a PR, new changes are applied to a local copy of the VMR, then it is source-built and tested + +parameters: + +- name: vmrPath + type: string + default: $(Agent.BuildDirectory)/vmr + +- name: vmrBranch + displayName: dotnet/dotnet branch to use + type: string + default: $(Build.SourceBranchName) + +# Resource id of the installer build to retrieve MSFT SDK from +- name: installerBuildResourceId + type: string + +- name: buildName + type: string + +- name: architecture + type: string + +- name: container + type: string + +- name: pool + type: object + +# Allow downloading artifacts from the internet during the build +- name: runOnline + type: boolean + +# Skip running the SDK content smoke-tests +- name: excludeSdkContentTests + type: boolean + default: false + +# Name of a previous job (from the same template as this) whose output will be used to build this job +# The SDK from its artifacts is copied to vmr/.dotnet +- name: reuseBuildArtifactsFrom + type: string + default: '' + +- name: excludeOmniSharpTests + type: boolean + +- name: bootstrapPrep + type: boolean + +- name: enablePoison + type: boolean + +jobs: +- job: ${{ parameters.buildName }}_${{ parameters.architecture }} + timeoutInMinutes: 150 + pool: ${{ parameters.pool }} + ${{ if ne(parameters.reuseBuildArtifactsFrom, '') }}: + # Always attempt to run the bootstrap leg (e.g. even when stage 1 tests fail) in order to get a complete accessment of the build status. + # The build shortcuts when stage 1 build fails and doesn't produce the SDK. + condition: succeededOrFailed() + dependsOn: ${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }} + variables: + - template: /eng/common/templates/variables/pool-providers.yml + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - group: DotNetBot-GitHub + - group: AzureDevOps-Artifact-Feeds-Pats + - ${{ else }}: + - name: BotAccount-dotnet-bot-repo-PAT + value: N/A + - ${{ if eq(parameters.reuseBuildArtifactsFrom, '') }}: + - name: additionalBuildArgs + value: '' + - ${{ else }}: + - name: additionalBuildArgs + value: ' --with-sdk /vmr/.dotnet' + + steps: + - template: ../steps/vmr-prepare.yml + parameters: + vmrBranch: ${{ parameters.vmrBranch }} + installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + skipComponentGovernanceDetection: true + + # Synchronize new content in the VMR during PRs (we expect this to come + - ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: + - template: ../steps/vmr-pull-updates.yml + parameters: + vmrPath: ${{ parameters.vmrPath }} + vmrBranch: ${{ parameters.vmrBranch }} + targetRef: $(Build.SourceVersion) # Synchronize the current installer commit + + - ${{ if ne(variables['System.TeamProject'], 'public') }}: + - script: cp ${{ parameters.vmrPath }}/src/installer/NuGet.config ${{ parameters.vmrPath }}/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config + displayName: Copy Test NuGet Config + + - task: Bash@3 + displayName: Setup Private Feeds Credentials + inputs: + filePath: ${{ parameters.vmrPath }}/src/installer/eng/common/SetupNugetSources.sh + arguments: ${{ parameters.vmrPath }}/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config $Token + env: + Token: $(dn-bot-dnceng-artifact-feeds-rw) + + - ${{ if ne(parameters.excludeSdkContentTests, 'true') }}: + - download: ${{ parameters.installerBuildResourceId }} + artifact: BlobArtifacts + patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-linux-${{ parameters.architecture }}.tar.gz' + displayName: Download MSFT SDK + + - ${{ if ne(parameters.reuseBuildArtifactsFrom, '') }}: + - download: current + artifact: ${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}_Artifacts + patterns: '*.tar.gz' + displayName: Download Previous Build + + - task: CopyFiles@2 + displayName: Copy Previous Build + inputs: + SourceFolder: $(Pipeline.Workspace)/${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}_Artifacts + Contents: '*.tar.gz' + TargetFolder: ${{ parameters.vmrPath }}/packages/archive/ + + - script: | + set -x + + if [[ -z '${{ parameters.reuseBuildArtifactsFrom }}' ]]; then + customPrepArgs= + if [[ '${{ parameters.bootstrapPrep }}' == 'True' ]]; then + customPrepArgs='--bootstrap' + fi + + docker run --rm -v ${{ parameters.vmrPath }}:/vmr -w /vmr ${{ parameters.container }} ./prep.sh ${customPrepArgs} + else + mkdir ${{ parameters.vmrPath }}/.dotnet + previousSdkPath="${{ parameters.vmrPath }}/packages/archive/dotnet-sdk-*.tar.gz" + eval tar -ozxf "$previousSdkPath" -C "${{ parameters.vmrPath }}/.dotnet" + eval rm -f "$previousSdkPath" + fi + displayName: Prep the Build + + - script: | + set -x + df -h + + networkArgs= + customBuildArgs= + if [[ '${{ parameters.runOnline }}' == 'True' ]]; then + customBuildArgs='--online' + else + networkArgs='--network none' + fi + + if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then + customBuildArgs="$customBuildArgs --poison" + fi + + docker run --rm -v ${{ parameters.vmrPath }}:/vmr -w /vmr ${networkArgs} ${{ parameters.container }} ./build.sh --clean-while-building $(additionalBuildArgs) ${customBuildArgs} + displayName: Build + + - script: | + set -x + + dockerVolumeArgs="-v ${{ parameters.vmrPath }}:/vmr" + dockerEnvArgs="-e SMOKE_TESTS_EXCLUDE_OMNISHARP=${{ parameters.excludeOmniSharpTests }} -e SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS=true" + poisonArg='' + + if [[ '${{ parameters.excludeSdkContentTests }}' != 'True' ]]; then + dockerVolumeArgs+=" -v $(Pipeline.Workspace)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/:/BlobArtifacts" + msftSdkTarballName=$(find "$(Pipeline.Workspace)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/" -name "dotnet-sdk-*-linux-${{ parameters.architecture }}.tar.gz" -exec basename {} \;) + dockerEnvArgs+=" -e SMOKE_TESTS_MSFT_SDK_TARBALL_PATH=/BlobArtifacts/$msftSdkTarballName" + fi + + if [[ '${{ parameters.enablePoison }}' == 'True' ]]; then + poisonArg='--poison' + dockerEnvArgs+=" -e SMOKE_TESTS_WARN_POISON_DIFFS=true" + fi + + docker run --rm $dockerVolumeArgs -w /vmr $dockerEnvArgs ${{ parameters.container }} ./build.sh $poisonArg --run-smoke-test $(additionalBuildArgs) -- -p:SmokeTestConsoleVerbosity=detailed + displayName: Run Tests + + # Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph. + - script: | + set -x + + targetFolder=$(Build.StagingDirectory)/BuildLogs/ + mkdir -p ${targetFolder} + + cd "${{ parameters.vmrPath }}" + find artifacts/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; + find artifacts/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \; + find artifacts/prebuilt-report/ -exec cp {} --parents -t ${targetFolder} \; + find src/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; + find src/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \; + find test/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; + find test/ -type f -name "Updated*.diff" -exec cp {} --parents -t ${targetFolder} \; + find test/ -type f -name "Updated*.txt" -exec cp {} --parents -t ${targetFolder} \; + displayName: Prepare BuildLogs staging directory + continueOnError: true + condition: succeededOrFailed() + + - publish: '$(Build.StagingDirectory)/BuildLogs' + artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) + displayName: Publish BuildLogs + continueOnError: true + condition: succeededOrFailed() + + - task: PublishTestResults@2 + displayName: Publish Test Results + condition: succeededOrFailed() + continueOnError: true + inputs: + testRunner: vSTest + testResultsFiles: 'test/**/*.trx' + searchFolder: ${{ parameters.vmrPath }} + mergeTestResults: true + publishRunAttachments: true + testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName) + + - publish: '${{ parameters.vmrPath }}/artifacts/${{ parameters.architecture }}/Release/' + artifact: $(Agent.JobName)_Artifacts + displayName: Publish Artifacts + condition: succeededOrFailed() + continueOnError: true diff --git a/eng/pipelines/vmr-synchronization.yml b/eng/pipelines/templates/jobs/vmr-synchronization.yml similarity index 52% rename from eng/pipelines/vmr-synchronization.yml rename to eng/pipelines/templates/jobs/vmr-synchronization.yml index cf624ab12..012b804a0 100644 --- a/eng/pipelines/vmr-synchronization.yml +++ b/eng/pipelines/templates/jobs/vmr-synchronization.yml @@ -1,8 +1,9 @@ -### This pipeline synchronizes code from product repositories into the VMR (https://github.com/dotnet/dotnet) +### This job synchronizes code from product repositories into the VMR (https://github.com/dotnet/dotnet) +### It synchronizes the content of the VMR to this new commit and pushes the changes parameters: - name: targetRef - displayName: Target revision in dotnet/installer to synchronize to + displayName: Target revision of dotnet/installer to synchronize type: string default: $(Build.SourceVersion) @@ -16,53 +17,42 @@ parameters: type: boolean default: false -trigger: - branches: - include: - - main - - release/* - - internal/release/* - -pr: none - -resources: - repositories: - - repository: vmr - type: github - name: dotnet/dotnet - endpoint: dotnet - -variables: -- template: /eng/common/templates/variables/pool-providers.yml -- template: templates/variables/vmr-variables.yml - jobs: -- job: Synchronize +- job: Synchronize_VMR displayName: Synchronize dotnet/dotnet timeoutInMinutes: 120 + variables: + - template: /eng/common/templates/variables/pool-providers.yml + - name: vmrPath + value: $(Agent.BuildDirectory)/vmr + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - group: DotNetBot-GitHub + - ${{ else }}: + - name: BotAccount-dotnet-bot-repo-PAT + value: N/A pool: ${{ if eq(variables['System.TeamProject'], 'public') }}: name: $(DncEngPublicBuildPool) demands: ImageOverride -equals Build.Ubuntu.2004.Amd64.Open - ${{ else }}: + ${{ if eq(variables['System.TeamProject'], 'internal') }}: name: $(DncEngInternalBuildPool) demands: ImageOverride -equals Build.Ubuntu.2004.Amd64 steps: - - template: templates/steps/vmr-prepare.yml + - template: ../steps/vmr-prepare.yml parameters: vmrBranch: ${{ parameters.vmrBranch }} - - template: templates/steps/vmr-pull-updates.yml + - template: ../steps/vmr-pull-updates.yml parameters: vmrPath: $(vmrPath) vmrBranch: ${{ parameters.vmrBranch }} targetRef: ${{ parameters.targetRef }} - vmrToken: $(BotAccount-dotnet-bot-repo-PAT) - - ${{ if and(not(parameters.noPush), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal')) }}: - - script: |- + # For official builds, push the changes to the VMR + - ${{ if and(not(parameters.noPush), not(in(variables['Build.Reason'], 'PullRequest')), eq(variables['System.TeamProject'], 'internal')) }}: + - script: | set -x git config --global user.email 'dotnet-maestro[bot]@users.noreply.github.com' && git config --global user.name 'dotnet-maestro[bot]' git remote add dotnet 'https://$(BotAccount-dotnet-bot-repo-PAT)@github.com/dotnet/dotnet.git' @@ -70,5 +60,14 @@ jobs: git branch ${{ parameters.vmrBranch }} git branch --set-upstream-to=dotnet/${{ parameters.vmrBranch }} ${{ parameters.vmrBranch }} || echo 'Branch ${{ parameters.vmrBranch }} not found in remote' git push dotnet ${{ parameters.vmrBranch }} + + # Export the SHA as an artifact to be reused by the official VMR build + sha=`git rev-parse HEAD` + mkdir -p "$(Build.ArtifactStagingDirectory)/VmrRevision" + echo $sha > "$(Build.ArtifactStagingDirectory)/VmrRevision/VmrRevision.txt" displayName: Push changes to dotnet/dotnet workingDirectory: $(vmrPath) + + - publish: $(Build.ArtifactStagingDirectory)/VmrRevision + artifact: VmrRevision + displayName: Publish new SHA diff --git a/eng/pipelines/templates/stages/vmr-build.yml b/eng/pipelines/templates/stages/vmr-build.yml new file mode 100644 index 000000000..4a3db6eaf --- /dev/null +++ b/eng/pipelines/templates/stages/vmr-build.yml @@ -0,0 +1,167 @@ +### This stage source-builds https://github.com/dotnet/dotnet with varying parameters +### If run in a PR, new changes are applied to a local copy of the VMR, then it is source-built and tested + +parameters: + # Resource id of the installer build to retrieve source/product tarball's from + installerBuildResourceId: current + + # Branch of the VMR to use (to push to for internal builds) + vmrBranch: $(Build.SourceBranchName) + + # The following parameters aren't expected to be passed in rather they are used for encapsulation + # ----------------------------------------------------------------------------------------------- + centOSStream8Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 + centOSStream9Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 + debian11Arm64Container: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-arm64v8 + fedora36Container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-36 + ubuntu2004Container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04 + + # Internal builds + poolInternalAmd64: + name: NetCore1ESPool-Svc-Internal + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + poolInternalAmd64PR: + name: NetCore1ESPool-Internal-XL + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 + poolInternalArm64: + name: Docker-Linux-Arm-Internal + + # Public builds / PRs + poolPublicAmd64: + name: NetCore-Public-XL + demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open + +stages: +- stage: VMR_Source_Build + displayName: VMR Source-Build + dependsOn: [] + variables: + - ${{ if eq(variables['System.TeamProject'], 'public') }}: + - name: defaultPoolName + value: ${{ parameters.poolPublicAmd64.name }} + - name: defaultPoolDemands + value: ${{ parameters.poolPublicAmd64.demands }} + - ${{ if eq(variables['System.TeamProject'], 'internal') }}: + - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: + - name: defaultPoolName + value: ${{ parameters.poolInternalAmd64PR.name }} + - name: defaultPoolDemands + value: ${{ parameters.poolInternalAmd64PR.demands }} + - ${{ else }}: + - name: defaultPoolName + value: ${{ parameters.poolInternalAmd64.name }} + - name: defaultPoolDemands + value: ${{ parameters.poolInternalAmd64.demands }} + + jobs: + - template: ../jobs/vmr-build.yml + parameters: + buildName: CentOSStream8_Online + installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + vmrBranch: ${{ parameters.vmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemands }} + container: ${{ parameters.centOSStream8Container }} + bootstrapPrep: false # 🚫 + enablePoison: false # 🚫 + excludeSdkContentTests: true # ✅ + excludeOmniSharpTests: true # ✅ + runOnline: true # ✅ + + - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: + - template: ../jobs/vmr-build.yml + parameters: + buildName: CentOSStream8_Offline + installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + vmrBranch: ${{ parameters.vmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemands }} + container: ${{ parameters.centOSStream8Container }} + bootstrapPrep: false # 🚫 + enablePoison: false # 🚫 + excludeSdkContentTests: false # 🚫 + excludeOmniSharpTests: true # ✅ + runOnline: false # 🚫 + + - template: ../jobs/vmr-build.yml + parameters: + buildName: CentOSStream9_Offline + installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + vmrBranch: ${{ parameters.vmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemands }} + container: ${{ parameters.centOSStream9Container }} + bootstrapPrep: false # 🚫 + enablePoison: false # 🚫 + excludeSdkContentTests: false # 🚫 + excludeOmniSharpTests: false # 🚫 + runOnline: false # 🚫 + + - template: ../jobs/vmr-build.yml + parameters: + buildName: Fedora36_Offline + installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + vmrBranch: ${{ parameters.vmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemands }} + container: ${{ parameters.fedora36Container }} + bootstrapPrep: false # 🚫 + enablePoison: true # ✅ + excludeSdkContentTests: false # 🚫 + excludeOmniSharpTests: false # 🚫 + runOnline: false # 🚫 + + - template: ../jobs/vmr-build.yml + parameters: + buildName: Ubuntu2004_Offline + installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + vmrBranch: ${{ parameters.vmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemands }} + container: ${{ parameters.ubuntu2004Container }} + bootstrapPrep: false # 🚫 + enablePoison: false # 🚫 + excludeSdkContentTests: false # 🚫 + excludeOmniSharpTests: false # 🚫 + runOnline: false # 🚫 + + - template: ../jobs/vmr-build.yml + parameters: + buildName: Debian11_Offline + installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + vmrBranch: ${{ parameters.vmrBranch }} + architecture: arm64 + pool: ${{ parameters.poolInternalArm64 }} + container: ${{ parameters.debian11Arm64Container }} + bootstrapPrep: true # ✅ + enablePoison: false # 🚫 + excludeSdkContentTests: false # 🚫 + excludeOmniSharpTests: false # 🚫 + runOnline: false # 🚫 + + - template: ../jobs/vmr-build.yml + parameters: + buildName: Fedora36_Offline_Using_Previous + installerBuildResourceId: ${{ parameters.installerBuildResourceId }} + vmrBranch: ${{ parameters.vmrBranch }} + architecture: x64 + pool: + name: ${{ variables.defaultPoolName }} + demands: ${{ variables.defaultPoolDemands }} + container: ${{ parameters.fedora36Container }} + bootstrapPrep: false # 🚫 + enablePoison: false # 🚫 + excludeSdkContentTests: true # ✅ + excludeOmniSharpTests: false # 🚫 + runOnline: false # 🚫 + reuseBuildArtifactsFrom: Fedora36_Offline diff --git a/eng/pipelines/templates/steps/vmr-prepare.yml b/eng/pipelines/templates/steps/vmr-prepare.yml index 5a18630f9..2786cbfad 100644 --- a/eng/pipelines/templates/steps/vmr-prepare.yml +++ b/eng/pipelines/templates/steps/vmr-prepare.yml @@ -7,21 +7,42 @@ parameters: type: string default: $(Build.SourceBranchName) +- name: installerBuildResourceId + displayName: Installer build to get the commit hash to check out from + type: string + default: current + +- name: skipComponentGovernanceDetection + type: boolean + default: false + steps: - checkout: vmr - clean: true displayName: Clone dotnet/dotnet - fetchDepth: 0 path: vmr + clean: true + fetchDepth: 0 # We need this so that we can check out the new commit -- script: | - git switch -c ${{ parameters.vmrBranch }} - displayName: Checkout ${{ parameters.vmrBranch }} - workingDirectory: $(Agent.BuildDirectory)/vmr +- ${{ if ne(parameters.installerBuildResourceId, 'current') }}: + - download: ${{ parameters.installerBuildResourceId }} + artifact: VmrRevision + displayName: Download VmrRevision + + - script: | + set -ex + sha=`head -n 1 $(Pipeline.Workspace)/${{ parameters.installerBuildResourceId }}/VmrRevision/VmrRevision.txt` + git checkout $sha + displayName: Check out dotnet/dotnet commit + workingDirectory: $(Agent.BuildDirectory)/vmr +- ${{ else }}: + - script: | + git switch -c ${{ parameters.vmrBranch }} + displayName: Checkout ${{ parameters.vmrBranch }} + workingDirectory: $(Agent.BuildDirectory)/vmr # TODO (https://github.com/dotnet/arcade/issues/11332): Allow full CG? # Currently, we ignore dirs of individual repos - they have been scanned before -- ${{ if and(ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal')) }}: +- ${{ if and(not(parameters.skipComponentGovernanceDetection), ne(variables['Build.Reason'], 'PullRequest'), eq(variables['System.TeamProject'], 'internal')) }}: - task: ComponentGovernanceComponentDetection@0 inputs: sourceScanPath: $(Agent.BuildDirectory)/vmr diff --git a/eng/pipelines/templates/steps/vmr-pull-updates.yml b/eng/pipelines/templates/steps/vmr-pull-updates.yml index 960f58975..0fb3ca76d 100644 --- a/eng/pipelines/templates/steps/vmr-pull-updates.yml +++ b/eng/pipelines/templates/steps/vmr-pull-updates.yml @@ -12,10 +12,6 @@ parameters: type: string default: $(Build.SourceVersion) -- name: vmrToken - displayName: PAT that allows pushing to dotnet/dotnet - type: string - - name: vmrPath displayName: Path where the dotnet/dotnet is checked out to type: string @@ -23,9 +19,7 @@ parameters: steps: - checkout: self - clean: true displayName: Clone dotnet/installer - fetchDepth: 0 path: installer - script: | @@ -37,22 +31,23 @@ steps: - script: git checkout -B ${{ parameters.vmrBranch }} displayName: Prepare branch ${{ parameters.vmrBranch }} - workingDirectory: $(vmrPath) + workingDirectory: ${{ parameters.vmrPath }} -# TODO (https://github.com/dotnet/arcade/issues/11386): Remove this step -# This step is here so that darc can find the PR commit (which might be in some fork) -# We need to make darc understand that it needs to look in the fork from this PR -- ${{ if eq(variables['Build.Reason'], 'PullRequest') }}: - - script: | - cp -r $(Agent.BuildDirectory)/installer $(Agent.TempDirectory)/03298978DFFFCD23 - displayName: Prepare dotnet/installer clone +# This step make sure that darc finds the PR commit (which might be in a fork of installer) +# The cloned folder does have the commit so we copy it into temp where the tooling will find it +# We also cannot keep the AzDO remote as pulling from it would require a token +- script: | + cp -r $(Agent.BuildDirectory)/installer $(Agent.TempDirectory)/03298978DFFFCD23 + cd $(Agent.TempDirectory)/03298978DFFFCD23 + git remote remove origin + git remote add origin https://github.com/dotnet/installer + displayName: Prepare dotnet/installer clone - script: > $(Agent.BuildDirectory)/installer/.dotnet/dotnet darc vmr update --vmr ${{ parameters.vmrPath }} --tmp $(Agent.TempDirectory) - --azdev-pat $(System.AccessToken) - --github-pat ${{ parameters.vmrToken }} + --github-pat $(BotAccount-dotnet-bot-repo-PAT) --recursive --verbose installer:${{ parameters.targetRef }} diff --git a/eng/pipelines/templates/variables/vmr-variables.yml b/eng/pipelines/templates/variables/vmr-variables.yml deleted file mode 100644 index 0d6af8fff..000000000 --- a/eng/pipelines/templates/variables/vmr-variables.yml +++ /dev/null @@ -1,9 +0,0 @@ -variables: -- name: vmrPath - value: $(Agent.BuildDirectory)/vmr - -- ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNetBot-GitHub -- ${{ else }}: - - name: BotAccount-dotnet-bot-repo-PAT - value: N/A \ No newline at end of file diff --git a/eng/pipelines/vmr-ci.yml b/eng/pipelines/vmr-ci.yml new file mode 100644 index 000000000..3d76618ab --- /dev/null +++ b/eng/pipelines/vmr-ci.yml @@ -0,0 +1,26 @@ +trigger: none +pr: none +resources: + pipelines: + - pipeline: installer-build-resource + source: dotnet-installer-official-ci + trigger: + branches: + include: + - main + - release/* + - internal/release/* + stages: + - Build + - VMR_Synchronization + + repositories: + - repository: vmr + type: github + name: dotnet/dotnet + endpoint: dotnet + +stages: +- template: /eng/pipelines/templates/stages/vmr-build.yml + parameters: + installerBuildResourceId: installer-build-resource diff --git a/eng/pipelines/vmr-validation.yml b/eng/pipelines/vmr-validation.yml deleted file mode 100644 index bdd0bbd31..000000000 --- a/eng/pipelines/vmr-validation.yml +++ /dev/null @@ -1,51 +0,0 @@ -### This pipeline validates new changes of the VMR (https://github.com/dotnet/dotnet) -### VMR is cloned and updates are applied locally - -parameters: -- name: vmrBranch - displayName: dotnet/dotnet branch to use - type: string - default: $(System.PullRequest.TargetBranch) - -trigger: none - -pr: - branches: - include: - - main - - release/* - - internal/release/* - -resources: - repositories: - - repository: vmr - type: github - name: dotnet/dotnet - endpoint: public - -variables: -- template: /eng/common/templates/variables/pool-providers.yml -- template: templates/variables/vmr-variables.yml - -jobs: -- job: Validate - displayName: Test VMR synchronization - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - name: $(DncEngPublicBuildPool) - demands: ImageOverride -equals Build.Ubuntu.2004.Amd64.Open - ${{ else }}: - name: $(DncEngInternalBuildPool) - demands: ImageOverride -equals Build.Ubuntu.2004.Amd64 - - steps: - - template: templates/steps/vmr-prepare.yml - parameters: - vmrBranch: ${{ parameters.vmrBranch }} - - - template: templates/steps/vmr-pull-updates.yml - parameters: - vmrPath: $(vmrPath) - vmrBranch: ${{ parameters.vmrBranch }} - targetRef: $(Build.SourceVersion) - vmrToken: $(BotAccount-dotnet-bot-repo-PAT) diff --git a/eng/source-build-tarball-build-official.yml b/eng/source-build-tarball-build-official.yml deleted file mode 100644 index 5d396baab..000000000 --- a/eng/source-build-tarball-build-official.yml +++ /dev/null @@ -1,26 +0,0 @@ -trigger: none -pr: none -resources: - pipelines: - - pipeline: installer-build-resource - source: dotnet-installer-official-ci - trigger: - branches: - include: - - main - - release/* - - internal/release/* - exclude: - - release/6.0.3xx - - internal/release/6.0.3xx - - release/6.0.4xx - - internal/release/6.0.4xx - stages: - - build - -stages: -- stage: build - jobs: - - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml - parameters: - installerBuildResourceId: installer-build-resource diff --git a/eng/source-build-tarball-build.yml b/eng/source-build-tarball-build.yml deleted file mode 100644 index da87d93cf..000000000 --- a/eng/source-build-tarball-build.yml +++ /dev/null @@ -1,14 +0,0 @@ -trigger: none -pr: none -resources: - pipelines: - - pipeline: installer-build-resource - source: installer - trigger: none - -stages: -- stage: build - jobs: - - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml - parameters: - installerBuildResourceId: installer-build-resource diff --git a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml deleted file mode 100644 index 042bb7349..000000000 --- a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml +++ /dev/null @@ -1,205 +0,0 @@ -# Builds a source-build tarball - -parameters: - # Architecture to build - architecture: - - # Custom condition to apply to the job - condition: - - # Dependent jobs that must be completed before this job will run - dependsOn: - - # Skip running the SDK content smoke-tests - excludeSdkContentTests: false - - # Resource id of the installer build to retrieve source/product tarball's from - installerBuildResourceId: - - # Matrix to run jobs over - matrix: - - # Job name - name: - - # Agent pool to use - pool: - - # Build using the previous source-build artifacts - usePreviousArtifacts: false - -jobs: -- job: ${{ parameters.name }} - condition: ${{ parameters.condition }} - dependsOn: ${{ parameters.dependsOn }} - pool: ${{ parameters.pool }} - strategy: - matrix: ${{ parameters.matrix }} - timeoutInMinutes: 300 - variables: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - group: AzureDevOps-Artifact-Feeds-Pats - - ${{ if eq(parameters.usePreviousArtifacts, 'true') }}: - - name: additionalBuildArgs - value: --with-sdk /tarball/.dotnet - - name: tarballDir - value: $(Build.StagingDirectory)/tarball - workspace: - clean: all - - steps: - - checkout: self - clean: true - - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: Bash@3 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - - download: ${{ parameters.installerBuildResourceId }} - artifact: BlobArtifacts - patterns: '**/dotnet-sdk-source*.tar.gz' - displayName: Download Source Tarball - - - ${{ if ne(parameters.excludeSdkContentTests, 'true') }}: - - download: ${{ parameters.installerBuildResourceId }} - artifact: BlobArtifacts - patterns: '**/dotnet-sdk-+([0-9]).+([0-9]).+([0-9])?(-@(alpha|preview|rc|rtm)*)-linux-${{ parameters.architecture }}.tar.gz' - displayName: Download MSFT sdk Tarball - - - ${{ if eq(parameters.usePreviousArtifacts, 'true') }}: - - download: current - artifact: $(_PreviousSourceBuildArtifact) - patterns: '*.tar.gz' - displayName: Download Previous Source Build Artifacts - - - task: CopyFiles@2 - displayName: Copy Previous Source Build Artifacts - inputs: - SourceFolder: $(PIPELINE.WORKSPACE)/$(_PreviousSourceBuildArtifact) - Contents: '*.tar.gz' - TargetFolder: $(tarballDir)/packages/archive/ - - - script: | - set -x - - resourceIdPathSegment= - if [[ '${{ parameters.installerBuildResourceId }}' != 'current' ]]; then - resourceIdPathSegment='${{ parameters.installerBuildResourceId }}/' - fi - - mkdir -p "$(tarballDir)" - tarballFilePath="$(PIPELINE.WORKSPACE)/${resourceIdPathSegment}BlobArtifacts/dotnet-sdk-source*.tar.gz" - eval tar -ozxf "$tarballFilePath" -C "$(tarballDir)" - eval rm -f "$tarballFilePath" - displayName: Extract Tarball - - - script: | - set -x - - if [[ '${{ parameters.usePreviousArtifacts }}' != 'true' ]]; then - customPrepArgs= - if [[ '$(_BootstrapPrep)' == 'true' ]]; then - customPrepArgs='--bootstrap' - fi - - docker run --rm -v $(tarballDir):/tarball -w /tarball $(_Container) ./prep.sh ${customPrepArgs} - else - mkdir $(tarballDir)/.dotnet - tarballFilePath="$(tarballDir)/packages/archive/dotnet-sdk-*.tar.gz" - eval tar -ozxf "$tarballFilePath" -C "$(tarballDir)/.dotnet" - eval rm -f "$tarballFilePath" - fi - displayName: Prep Tarball - - - script: | - set -x - df -h - - networkArgs= - customBuildArgs= - if [[ '$(_RunOnline)' == 'true' ]]; then - customBuildArgs='--online' - else - networkArgs='--network none' - fi - - if [[ '$(_EnablePoison)' == 'true' ]]; then - customBuildArgs="$customBuildArgs --poison" - fi - - docker run --rm -v $(tarballDir):/tarball -w /tarball ${networkArgs} $(_Container) ./build.sh --clean-while-building ${customBuildArgs} $(additionalBuildArgs) - displayName: Build Tarball - - - script: | - set -x - - # Use installer repo's NuGet.config during online testing to utilize internal feeds - rm -f $(tarballDir)/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config - cp $(Build.SourcesDirectory)/NuGet.config $(tarballDir)/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config - - dockerVolumeArgs="-v $(tarballDir):/tarball" - dockerEnvArgs="-e SMOKE_TESTS_EXCLUDE_OMNISHARP=$(_ExcludeOmniSharpTests) -e SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS=true" - poisonArg='' - - if [[ '${{ parameters.excludeSdkContentTests}}' != 'true' ]]; then - dockerVolumeArgs+=" -v $(PIPELINE.WORKSPACE)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/:/BlobArtifacts" - msftSdkTarballName=$(find "$(PIPELINE.WORKSPACE)/${{ parameters.installerBuildResourceId }}/BlobArtifacts/" -name "dotnet-sdk-*-linux-${{ parameters.architecture }}.tar.gz" -exec basename {} \;) - dockerEnvArgs+=" -e SMOKE_TESTS_MSFT_SDK_TARBALL_PATH=/BlobArtifacts/$msftSdkTarballName" - fi - - if [[ '$(_EnablePoison)' == 'true' ]]; then - poisonArg='--poison' - dockerEnvArgs+=" -e SMOKE_TESTS_WARN_POISON_DIFFS=true" - fi - - docker run --rm $dockerVolumeArgs -w /tarball $dockerEnvArgs $(_Container) ./build.sh $poisonArg --run-smoke-test $(additionalBuildArgs) -- -p:SmokeTestConsoleVerbosity=detailed - displayName: Run Tests - - # Don't use CopyFiles@2 as it encounters permissions issues because it indexes all files in the source directory graph. - - script: | - set -x - - targetFolder=$(Build.StagingDirectory)/BuildLogs/ - mkdir -p ${targetFolder} - - cd "$(tarballDir)" - find artifacts/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; - find artifacts/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \; - find artifacts/prebuilt-report/ -exec cp {} --parents -t ${targetFolder} \; - find src/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; - find src/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \; - find test/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; - find test/ -type f -name "Updated*.diff" -exec cp {} --parents -t ${targetFolder} \; - find test/ -type f -name "Updated*.txt" -exec cp {} --parents -t ${targetFolder} \; - displayName: Prepare BuildLogs staging directory - continueOnError: true - condition: succeededOrFailed() - - - publish: '$(Build.StagingDirectory)/BuildLogs' - artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) - displayName: Publish BuildLogs - continueOnError: true - condition: succeededOrFailed() - - - task: PublishTestResults@2 - displayName: Publish Test Results - condition: succeededOrFailed() - continueOnError: true - inputs: - testRunner: vSTest - testResultsFiles: 'test/**/*.trx' - searchFolder: $(tarballDir) - mergeTestResults: true - publishRunAttachments: true - testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName) - - - publish: '$(tarballDir)/artifacts/${{ parameters.architecture}}/Release/' - artifact: $(Agent.JobName)_Artifacts - displayName: Publish Artifacts - condition: succeededOrFailed() - continueOnError: true diff --git a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml deleted file mode 100644 index acdfe5901..000000000 --- a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-create-tarball.yml +++ /dev/null @@ -1,85 +0,0 @@ -# Creates a source-build tarball - -jobs: -- job: Source_Build_Create_Tarball - container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-36 - displayName: Source-Build Create Tarball - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - name: NetCore-Public-XL - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - ${{ else }}: - name: NetCore-Svc-Public - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - name: NetCore1ESPool-Internal-XL - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - ${{ else }}: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - - variables: - - name: _BuildConfig - value: Release - - - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - group: DotNet-Source-Build-All-Orgs-Source-Access - workspace: - clean: all - - steps: - - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - task: Bash@3 - displayName: Setup Private Feeds Credentials - inputs: - filePath: $(Build.SourcesDirectory)/eng/common/SetupNugetSources.sh - arguments: $(Build.SourcesDirectory)/NuGet.config $Token - env: - Token: $(dn-bot-dnceng-artifact-feeds-rw) - - script: | - set -x - df -h - - officialBuildArgs= - if [ '${{ and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}' = 'True' ]; then - officialBuildArgs='/p:OfficialBuildId=$(BUILD.BUILDNUMBER)' - fi - - # If building on the internal project, the internal storage variable may be available (usually only if needed) - # In that case, add variables to allow the download of internal runtimes if the specified versions are not found - # in the default public locations. - internalRuntimeDownloadArgs= - if [ '$(dotnetbuilds-internal-container-read-token-base64)' != '$''(dotnetbuilds-internal-container-read-token-base64)' ]; then - internalRuntimeDownloadArgs='/p:DotNetRuntimeSourceFeed=https://dotnetbuilds.blob.core.windows.net/internal /p:DotNetRuntimeSourceFeedKey=$(dotnetbuilds-internal-container-read-token-base64) --runtimesourcefeed https://dotnetbuilds.blob.core.windows.net/internal --runtimesourcefeedkey $(dotnetbuilds-internal-container-read-token-base64)' - fi - - ./build.sh \ - --ci \ - --configuration $(_BuildConfig) \ - --publish \ - -bl \ - $officialBuildArgs \ - $internalRuntimeDownloadArgs \ - /p:DotNetPublishUsingPipelines=true \ - /p:ArcadeBuildTarball=true \ - /p:AzDoPat=$(dn-bot-all-orgs-build-rw-code-rw) - displayName: Create Tarball - - - task: CopyFiles@2 - displayName: Prepare BuildLogs staging directory - inputs: - Contents: | - **/*.log - **/*.binlog - TargetFolder: '$(Build.StagingDirectory)/BuildLogs' - CleanTargetFolder: true - continueOnError: true - condition: succeededOrFailed() - - - publish: '$(Build.StagingDirectory)/BuildLogs' - artifact: $(Agent.JobName)_BuildLogs_Attempt$(System.JobAttempt) - displayName: Publish BuildLogs - continueOnError: true - condition: succeededOrFailed() diff --git a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml deleted file mode 100644 index c794bbc8f..000000000 --- a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml +++ /dev/null @@ -1,121 +0,0 @@ -# Builds a source-build tarball - -parameters: - # Dependent jobs that must be completed before this job will run - dependsOn: - - # Resource id of the installer build to retrieve source/product tarball's from - installerBuildResourceId: current - - # The following parameters aren't expected to be passed in rather they are used for encapsulation - # ----------------------------------------------------------------------------------------------- - centOSStream8Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream8 - centOSStream9Container: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-stream9 - debian11Arm64Container: mcr.microsoft.com/dotnet-buildtools/prereqs:debian-11-arm64v8 - fedora36Container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-36 - ubuntu2004Container: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-20.04 - poolInternalAmd64: - name: NetCore1ESPool-Svc-Internal - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - poolInternalAmd64PR: - name: NetCore1ESPool-Internal-XL - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64 - poolInternalArm64: - name: Docker-Linux-Arm-Internal - poolPublicAmd64: - name: NetCore-Public-XL - demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open - -jobs: -- template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml - parameters: - architecture: x64 - dependsOn: ${{ parameters.dependsOn }} - ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}: - excludeSdkContentTests: true - installerBuildResourceId: ${{ parameters.installerBuildResourceId }} - matrix: - CentOSStream8-Online: - _BootstrapPrep: false - _Container: ${{ parameters.centOSStream8Container }} - _EnablePoison: false - _ExcludeOmniSharpTests: true - _RunOnline: true - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - CentOSStream8-Offline: - _BootstrapPrep: false - _Container: ${{ parameters.centOSStream8Container }} - _EnablePoison: false - _ExcludeOmniSharpTests: true - _RunOnline: false - CentOSStream9-Offline: - _BootstrapPrep: false - _Container: ${{ parameters.centOSStream9Container }} - _EnablePoison: false - _ExcludeOmniSharpTests: false - _RunOnline: false - Fedora36-Offline: - _BootstrapPrep: false - _Container: ${{ parameters.fedora36Container }} - _EnablePoison: true - _ExcludeOmniSharpTests: false - _RunOnline: false - Ubuntu2004-Offline: - _BootstrapPrep: false - _Container: ${{ parameters.ubuntu2004Container }} - _EnablePoison: false - _ExcludeOmniSharpTests: false - _RunOnline: false - name: Build_Tarball_x64 - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - ${{ parameters.poolPublicAmd64 }} - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - ${{ parameters.poolInternalAmd64PR }} - ${{ else }}: - ${{ parameters.poolInternalAmd64 }} - -- ${{ if eq(variables['System.TeamProject'], 'internal') }}: - - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml - parameters: - architecture: arm64 - dependsOn: ${{ parameters.dependsOn }} - ${{ if in(variables['Build.Reason'], 'PullRequest') }}: - excludeSdkContentTests: true - installerBuildResourceId: ${{ parameters.installerBuildResourceId }} - matrix: - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - Debian11-Offline: - _BootstrapPrep: true - _Container: ${{ parameters.debian11Arm64Container }} - _EnablePoison: false - _ExcludeOmniSharpTests: false - _RunOnline: false - name: Build_Tarball_arm64 - pool: ${{ parameters.poolInternalArm64 }} - -- ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - - template: /src/SourceBuild/Arcade/eng/common/templates/job/source-build-build-tarball.yml - parameters: - architecture: x64 - # Always attempt to run the bootstrap leg (e.g. even when stage 1 tests fail) in order to get a complete accessment of the build status. - # The bootstrap build will shortcut if the stage 1 build failed. - condition: succeededOrFailed() - dependsOn: Build_Tarball_x64 - excludeSdkContentTests: true - installerBuildResourceId: ${{ parameters.installerBuildResourceId }} - matrix: - Fedora36-Offline: - _PreviousSourceBuildArtifact: Build_Tarball_x64 Fedora36-Offline_Artifacts - _Container: ${{ parameters.fedora36Container }} - _EnablePoison: false - _ExcludeOmniSharpTests: false - _RunOnline: false - name: Build_Tarball_x64_Using_Previous - pool: - ${{ if eq(variables['System.TeamProject'], 'public') }}: - ${{ parameters.poolPublicAmd64 }} - ${{ if eq(variables['System.TeamProject'], 'internal') }}: - ${{ parameters.poolInternalAmd64 }} - usePreviousArtifacts: true From 3b7a642e1484ff070a3a5d5eb26cddb115c75ce0 Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" <42748379+dotnet-maestro[bot]@users.noreply.github.com> Date: Thu, 1 Dec 2022 09:23:34 -0600 Subject: [PATCH 9/9] Update dependencies from https://github.com/dotnet/source-build-externals build 20221130.1 (#15075) Microsoft.SourceBuild.Intermediate.source-build-externals From Version 8.0.0-alpha.1.22579.1 -> To Version 8.0.0-alpha.1.22580.1 Co-authored-by: dotnet-maestro[bot] --- eng/Version.Details.xml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 21b90cf27..64f941ed6 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -180,9 +180,9 @@ c3ad00ae84489071080a606f6a8e43c9a91a5cc2 - + https://github.com/dotnet/source-build-externals - dd9d921db96b4469ebac73ee708cce8a31845c0c + fd96997313778d5e9750d55cbb580e05bcc38465