diff --git a/.config/dotnet-tools.json b/.config/dotnet-tools.json index c91757e55..c1a7184c8 100644 --- a/.config/dotnet-tools.json +++ b/.config/dotnet-tools.json @@ -3,7 +3,7 @@ "isRoot": true, "tools": { "microsoft.dotnet.darc": { - "version": "1.1.0-beta.23059.1", + "version": "1.1.0-beta.23062.1", "commands": [ "darc" ] diff --git a/NuGet.config b/NuGet.config index cb866adcb..77050d5f1 100644 --- a/NuGet.config +++ b/NuGet.config @@ -11,12 +11,8 @@ - - - - - + diff --git a/README.md b/README.md index 83cceea51..620c58fc7 100644 --- a/README.md +++ b/README.md @@ -60,7 +60,13 @@ For the latest information about Source-Build support for new .NET versions, ple The dependencies for building .NET from source can be found [here](https://github.com/dotnet/runtime/blob/main/docs/workflow/requirements/linux-requirements.md). -## Building +## Building .NET 8.0 + +.NET 8.0 (currently in prerelease) and newer will be built from the [dotnet/dotnet](https://github.com/dotnet/dotnet) repo. +Clone the dotnet/dotnet repo and check out the tag for the desired release. +Then, follow the instructions in [dotnet/dotnet's README](https://github.com/dotnet/dotnet/blob/main/README.md#dev-instructions) to build .NET from source. + +## Building .NET 7.0 and .NET 6.0 1. Create a .NET source tarball. diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index b59782f59..a30fe8929 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -167,22 +167,22 @@ https://github.com/Microsoft/ApplicationInsights-dotnet 53b80940842204f78708a538628288ff5d741a1d - + https://github.com/dotnet/emsdk - c91d3e0511d3888e5c6ddc096a224ec6c141beed + 5578fc4c0d1f4a1a01ad2801de2852cbe422b9ab - + https://github.com/dotnet/emsdk - c91d3e0511d3888e5c6ddc096a224ec6c141beed + 5578fc4c0d1f4a1a01ad2801de2852cbe422b9ab https://github.com/dotnet/deployment-tools c3ad00ae84489071080a606f6a8e43c9a91a5cc2 - + https://github.com/dotnet/source-build-externals - 54fb15633d58bf41248a7ea4bf366b4ea4798b89 + df9dd3b66b2fa6d86eb01bb5b2493435bdfa2649 @@ -205,13 +205,13 @@ https://github.com/dotnet/arcade 5a5ae41c42e475e280342b668e668ff34539e36d - + https://github.com/dotnet/arcade-services - 60e937ba1a05bfa01ce51a4fe0a10fa3f5b026f8 + 068e0045e2903fb9a1ac09630c28537f09ec68d8 - + https://github.com/dotnet/arcade-services - 60e937ba1a05bfa01ce51a4fe0a10fa3f5b026f8 + 068e0045e2903fb9a1ac09630c28537f09ec68d8 https://github.com/dotnet/runtime diff --git a/eng/Versions.props b/eng/Versions.props index 14bf70a57..745af34d2 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -30,7 +30,7 @@ - 1.1.0-beta.23059.1 + 1.1.0-beta.23062.1 @@ -147,14 +147,14 @@ $(NUnit3DotNetNewTemplatePackageVersion) $(MicrosoftDotNetCommonItemTemplates70PackageVersion) $(MicrosoftDotNetCommonItemTemplates70PackageVersion) - 7.0.0 + 7.0.2 $(MicrosoftWinFormsProjectTemplates60PackageVersion) $(MicrosoftWPFProjectTemplates60PackageVersion) $(NUnit3DotNetNewTemplatePackageVersion) $(MicrosoftDotNetCommonItemTemplates60PackageVersion) $(MicrosoftDotNetCommonItemTemplates60PackageVersion) - 6.0.11 + 6.0.13 $(MicrosoftWinFormsProjectTemplates50PackageVersion) $(MicrosoftWPFProjectTemplates50PackageVersion) @@ -168,7 +168,7 @@ 1.7.2 3.1.27 $(MicrosoftDotNetCommonItemTemplates31PackageVersion) - 3.1.31 + 3.1.32 3.2.1 4.8.0-rc2.19462.10 @@ -200,8 +200,8 @@ 12.3.2372 16.0.1478 $(MicrosoftNETCoreAppRefPackageVersion) - 8.0.0-alpha.1.23054.1 - 8.0.0-alpha.1.23054.1 + 8.0.0-alpha.1.23061.1 + 8.0.0-alpha.1.23061.1 $(MicrosoftNETWorkloadEmscriptennet7Manifest80100Version) diff --git a/eng/pipelines/templates/jobs/vmr-build.yml b/eng/pipelines/templates/jobs/vmr-build.yml index 5c21c1bf5..eca9930ba 100644 --- a/eng/pipelines/templates/jobs/vmr-build.yml +++ b/eng/pipelines/templates/jobs/vmr-build.yml @@ -51,6 +51,10 @@ parameters: - name: enablePoison type: boolean +# Instead of building the VMR directly, exports the sources into a tarball and builds from that +- name: buildFromArchive + type: boolean + jobs: - job: ${{ parameters.buildName }}_${{ parameters.architecture }} timeoutInMinutes: 150 @@ -75,6 +79,15 @@ jobs: - name: additionalBuildArgs value: ' --with-sdk /vmr/.dotnet' + # Location of the VMR sources + # We either build the repo directly, or we extract them outside (which is what partners do) + - ${{ if parameters.buildFromArchive }}: + - name: sourcesPath + value: $(Build.StagingDirectory)/dotnet-sources/ + - ${{ else }}: + - name: sourcesPath + value: ${{ parameters.vmrPath }} + steps: - template: ../steps/vmr-prepare.yml parameters: @@ -90,15 +103,24 @@ jobs: vmrBranch: ${{ parameters.vmrBranch }} targetRef: $(Build.SourceVersion) # Synchronize the current installer commit + - ${{ if parameters.buildFromArchive }}: + - script: | + set -ex + "${{ parameters.vmrPath }}/eng/pack-sources.sh" -o "$(Build.StagingDirectory)/dotnet-sources.tar.gz" + tar -xf dotnet-sources.tar.gz + mv "dotnet-$(Build.SourceVersion)" "$(sourcesPath)" + displayName: Export VMR sources + workingDirectory: $(Build.StagingDirectory) + - ${{ if ne(variables['System.TeamProject'], 'public') }}: - - script: cp ${{ parameters.vmrPath }}/src/installer/NuGet.config ${{ parameters.vmrPath }}/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config + - script: cp "$(sourcesPath)/src/installer/NuGet.config" "$(sourcesPath)/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config" displayName: Copy Test NuGet Config - task: Bash@3 displayName: Setup Private Feeds Credentials inputs: - filePath: ${{ parameters.vmrPath }}/src/installer/eng/common/SetupNugetSources.sh - arguments: ${{ parameters.vmrPath }}/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config $Token + filePath: $(sourcesPath)/src/installer/eng/common/SetupNugetSources.sh + arguments: $(sourcesPath)/test/Microsoft.DotNet.SourceBuild.SmokeTests/assets/online.NuGet.Config $Token env: Token: $(dn-bot-dnceng-artifact-feeds-rw) @@ -119,7 +141,7 @@ jobs: inputs: SourceFolder: $(Pipeline.Workspace)/${{ parameters.reuseBuildArtifactsFrom }}_${{ parameters.architecture }}_Artifacts Contents: '*.tar.gz' - TargetFolder: ${{ parameters.vmrPath }}/packages/archive/ + TargetFolder: ${{ variables.sourcesPath }}/packages/archive/ - script: | set -x @@ -130,11 +152,11 @@ jobs: customPrepArgs='--bootstrap' fi - docker run --rm -v ${{ parameters.vmrPath }}:/vmr -w /vmr ${{ parameters.container }} ./prep.sh ${customPrepArgs} + docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${{ parameters.container }} ./prep.sh ${customPrepArgs} else - mkdir ${{ parameters.vmrPath }}/.dotnet - previousSdkPath="${{ parameters.vmrPath }}/packages/archive/dotnet-sdk-*.tar.gz" - eval tar -ozxf "$previousSdkPath" -C "${{ parameters.vmrPath }}/.dotnet" + mkdir $(sourcesPath)/.dotnet + previousSdkPath="$(sourcesPath)/packages/archive/dotnet-sdk-*.tar.gz" + eval tar -ozxf "$previousSdkPath" -C "$(sourcesPath)/.dotnet" eval rm -f "$previousSdkPath" fi displayName: Prep the Build @@ -155,13 +177,13 @@ jobs: customBuildArgs="$customBuildArgs --poison" fi - docker run --rm -v ${{ parameters.vmrPath }}:/vmr -w /vmr ${networkArgs} ${{ parameters.container }} ./build.sh --clean-while-building $(additionalBuildArgs) ${customBuildArgs} + docker run --rm -v "$(sourcesPath):/vmr" -w /vmr ${networkArgs} ${{ parameters.container }} ./build.sh --clean-while-building $(additionalBuildArgs) ${customBuildArgs} displayName: Build - script: | set -x - dockerVolumeArgs="-v ${{ parameters.vmrPath }}:/vmr" + dockerVolumeArgs="-v $(sourcesPath):/vmr" dockerEnvArgs="-e SMOKE_TESTS_EXCLUDE_OMNISHARP=${{ parameters.excludeOmniSharpTests }} -e SMOKE_TESTS_WARN_SDK_CONTENT_DIFFS=true" poisonArg='' @@ -186,7 +208,7 @@ jobs: targetFolder=$(Build.StagingDirectory)/BuildLogs/ mkdir -p ${targetFolder} - cd "${{ parameters.vmrPath }}" + cd "$(sourcesPath)" find artifacts/ -type f -name "*.binlog" -exec cp {} --parents -t ${targetFolder} \; find artifacts/ -type f -name "*.log" -exec cp {} --parents -t ${targetFolder} \; find artifacts/prebuilt-report/ -exec cp {} --parents -t ${targetFolder} \; @@ -212,12 +234,12 @@ jobs: inputs: testRunner: vSTest testResultsFiles: 'test/**/*.trx' - searchFolder: ${{ parameters.vmrPath }} + searchFolder: ${{ variables.sourcesPath }} mergeTestResults: true publishRunAttachments: true testRunTitle: SourceBuild_SmokeTests_$(Agent.JobName) - - publish: '${{ parameters.vmrPath }}/artifacts/${{ parameters.architecture }}/Release/' + - publish: '${{ variables.sourcesPath }}/artifacts/${{ parameters.architecture }}/Release/' artifact: $(Agent.JobName)_Artifacts displayName: Publish Artifacts condition: succeededOrFailed() diff --git a/eng/pipelines/templates/stages/vmr-build.yml b/eng/pipelines/templates/stages/vmr-build.yml index 8d052af5d..ba6a74ebf 100644 --- a/eng/pipelines/templates/stages/vmr-build.yml +++ b/eng/pipelines/templates/stages/vmr-build.yml @@ -68,11 +68,12 @@ stages: name: ${{ variables.defaultPoolName }} demands: ${{ variables.defaultPoolDemands }} container: ${{ parameters.centOSStream8Container }} - bootstrapPrep: false # 🚫 - enablePoison: false # 🚫 - excludeSdkContentTests: true # ✅ - excludeOmniSharpTests: true # ✅ - runOnline: true # ✅ + bootstrapPrep: false # 🚫 + enablePoison: false # 🚫 + excludeSdkContentTests: true # ✅ + excludeOmniSharpTests: true # ✅ + runOnline: true # ✅ + buildFromArchive: false # 🚫 - ${{ if ne(variables['Build.Reason'], 'PullRequest') }}: - template: ../jobs/vmr-build.yml @@ -90,6 +91,7 @@ stages: excludeSdkContentTests: false # 🚫 excludeOmniSharpTests: true # ✅ runOnline: false # 🚫 + buildFromArchive: true # ✅ - template: ../jobs/vmr-build.yml parameters: @@ -106,6 +108,7 @@ stages: excludeSdkContentTests: false # 🚫 excludeOmniSharpTests: false # 🚫 runOnline: false # 🚫 + buildFromArchive: true # ✅ - template: ../jobs/vmr-build.yml parameters: @@ -122,6 +125,7 @@ stages: excludeSdkContentTests: false # 🚫 excludeOmniSharpTests: false # 🚫 runOnline: false # 🚫 + buildFromArchive: true # ✅ - template: ../jobs/vmr-build.yml parameters: @@ -138,6 +142,7 @@ stages: excludeSdkContentTests: false # 🚫 excludeOmniSharpTests: false # 🚫 runOnline: false # 🚫 + buildFromArchive: false # 🚫 - template: ../jobs/vmr-build.yml parameters: @@ -152,6 +157,7 @@ stages: excludeSdkContentTests: false # 🚫 excludeOmniSharpTests: false # 🚫 runOnline: false # 🚫 + buildFromArchive: false # 🚫 - template: ../jobs/vmr-build.yml parameters: @@ -168,4 +174,5 @@ stages: excludeSdkContentTests: true # ✅ excludeOmniSharpTests: false # 🚫 runOnline: false # 🚫 + buildFromArchive: false # 🚫 reuseBuildArtifactsFrom: Fedora36_Offline diff --git a/eng/vmr-sync.sh b/eng/vmr-sync.sh index aa747f7a0..715ab3072 100755 --- a/eng/vmr-sync.sh +++ b/eng/vmr-sync.sh @@ -83,7 +83,7 @@ function highlight () { echo "${COLOR_CYAN}$FAILURE_PREFIX${1//${COLOR_RESET}/${COLOR_CYAN}}${COLOR_CLEAR}" } -installer_dir="$scriptroot/../" +installer_dir=$(realpath "$scriptroot/../") tmp_dir='' vmr_dir='' vmr_branch='main' @@ -179,10 +179,6 @@ fi highlight "Starting the synchronization to '$target_ref'.." set +e -# Temporary workaround while we fix fetching commits -rm -rf "$tmp_dir/installer" -cp -r "$installer_dir" "$tmp_dir/installer" - if "$dotnet" darc vmr update --vmr "$vmr_dir" --tmp "$tmp_dir" --$verbosity --recursive --additional-remotes "installer:$installer_dir" "installer:$target_ref"; then highlight "Synchronization succeeded" else diff --git a/src/SourceBuild/content/.devcontainer/devcontainer.json b/src/SourceBuild/content/.devcontainer/devcontainer.json index 43951e419..8f8e2ac2d 100644 --- a/src/SourceBuild/content/.devcontainer/devcontainer.json +++ b/src/SourceBuild/content/.devcontainer/devcontainer.json @@ -3,7 +3,7 @@ "name": "Default", "image": "mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-36", "hostRequirements": { - // A completely built .NET source-tarball is >64 GB + // A completely source built .NET is >64 GB with all the repos/artifacts "storage": "128gb" }, "customizations": { diff --git a/src/SourceBuild/content/.devcontainer/prebuilt-sdk/devcontainer.json b/src/SourceBuild/content/.devcontainer/prebuilt-sdk/devcontainer.json index fda4386e2..40d60fb29 100644 --- a/src/SourceBuild/content/.devcontainer/prebuilt-sdk/devcontainer.json +++ b/src/SourceBuild/content/.devcontainer/prebuilt-sdk/devcontainer.json @@ -3,7 +3,7 @@ "name": "Pre-built .NET SDK", "image": "mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-36", "hostRequirements": { - // A completely built .NET source-tarball is >64 GB + // A completely source built .NET is >64 GB with all the repos/artifacts "storage": "128gb" }, "customizations": { diff --git a/src/SourceBuild/content/CONTRIBUTING.md b/src/SourceBuild/content/CONTRIBUTING.md new file mode 100644 index 000000000..1b23b661b --- /dev/null +++ b/src/SourceBuild/content/CONTRIBUTING.md @@ -0,0 +1,7 @@ +Contributing +============ + +See [dotnet/runtime](./src/runtime/CONTRIBUTING.md) for general contribution guidelines such as allowed licenses. + +At this time, the VMR will not accept any changes and is a read-only mirror of the development repositories only. +Please, make the changes in the respective development repositories (e.g., [dotnet/runtime](https://github.com/dotnet/runtime) or [dotnet/sdk](https://github.com/dotnet/sdk)) and they will get synchronized into the VMR automatically. \ No newline at end of file diff --git a/src/SourceBuild/content/Directory.Build.props b/src/SourceBuild/content/Directory.Build.props index 3fe574c31..f8977b54b 100644 --- a/src/SourceBuild/content/Directory.Build.props +++ b/src/SourceBuild/content/Directory.Build.props @@ -119,9 +119,6 @@ $(LocalBlobStorageRoot)Sdk/ $(LocalBlobStorageRoot)Runtime/ $(LocalBlobStorageRoot)aspnetcore/Runtime/ - $(IntermediatePath)RestoreSources.props - $(IntermediatePath)PackageVersions.props - $(IntermediatePath)CurrentSourceBuiltPackageVersions.props $(BaseOutputPath)logs/ $(BaseOutputPath)msbuild-debug/ $(BaseOutputPath)roslyn-debug/ diff --git a/src/SourceBuild/content/build.sh b/src/SourceBuild/content/build.sh index a7d8113d4..e61ce1b25 100755 --- a/src/SourceBuild/content/build.sh +++ b/src/SourceBuild/content/build.sh @@ -102,19 +102,14 @@ fi if [ -f "${packagesArchiveDir}archiveArtifacts.txt" ]; then ARCHIVE_ERROR=0 if [ ! -d "$SCRIPT_ROOT/.dotnet" ] && [ "$CUSTOM_SDK_DIR" == "" ]; then - echo "ERROR: SDK not found at $SCRIPT_ROOT/.dotnet" + echo "ERROR: SDK not found at '$SCRIPT_ROOT/.dotnet'. Either run prep.sh to acquire one or specify one via the --with-sdk parameter." ARCHIVE_ERROR=1 fi if [ ! -f ${packagesArchiveDir}Private.SourceBuilt.Artifacts*.tar.gz ] && [ "$CUSTOM_PACKAGES_DIR" == "" ]; then - echo "ERROR: Private.SourceBuilt.Artifacts artifact not found at $packagesArchiveDir - Either run prep.sh or pass --with-packages parameter" + echo "ERROR: Private.SourceBuilt.Artifacts artifact not found at '$packagesArchiveDir'. Either run prep.sh to acquire it or specify one via the --with-packages parameter." ARCHIVE_ERROR=1 fi if [ $ARCHIVE_ERROR == 1 ]; then - echo "" - echo " Errors detected in tarball. To prep the tarball, run prep.sh while online to install an SDK" - echo " and Private.SourceBuilt.Artifacts tarball. After prepping the tarball, the tarball can be" - echo " built offline. As an alternative to prepping the tarball, these assets can be provided using" - echo " the --with-sdk and --with-packages parameters" exit 1 fi fi @@ -161,7 +156,7 @@ if [[ $arcadeSdkLine =~ $versionPattern ]]; then export ARCADE_BOOTSTRAP_VERSION=${BASH_REMATCH[1]} # Ensure that by default, the bootstrap version of the Arcade SDK is used. Source-build infra - # projects use bootstrap Arcade SDK, and would fail to find it in the tarball build. The repo + # projects use bootstrap Arcade SDK, and would fail to find it in the build. The repo # projects overwrite this so that they use the source-built Arcade SDK instad. export SOURCE_BUILT_SDK_ID_ARCADE=Microsoft.DotNet.Arcade.Sdk export SOURCE_BUILT_SDK_VERSION_ARCADE=$ARCADE_BOOTSTRAP_VERSION diff --git a/src/SourceBuild/content/eng/bootstrap/README.template.md b/src/SourceBuild/content/eng/bootstrap/README.template.md index e79c35fc4..f6d91d55f 100644 --- a/src/SourceBuild/content/eng/bootstrap/README.template.md +++ b/src/SourceBuild/content/eng/bootstrap/README.template.md @@ -139,6 +139,12 @@ ln -s $HOME/.dotnet/dotnet /usr/bin/dotnet You can also utilize [GitHub Codespaces](https://github.com/features/codespaces) where you can find preset containers in this repository. +### Exporting a source archive + +In case you'd like to export a more lightweight archive of sources that can be built outside of this git repository, a simple copy of the working tree won't do. +The build is using some git metadata (information from the `.git` directory) that are needed to be kept with the sources. +To export a `tar.gz` archive of the sources, you need to use the `eng/pack-sources.sh` script from within a clone of the VMR checked out at the revision that you're interested in. + ## List of components To enable full offline source-building of the VMR, we have no other choice than to synchronize all the necessary code into the VMR. This also includes any code referenced via git submodules. More details on why and how this is done can be found here: diff --git a/src/SourceBuild/content/eng/pack-sources.sh b/src/SourceBuild/content/eng/pack-sources.sh new file mode 100755 index 000000000..ecb2c8920 --- /dev/null +++ b/src/SourceBuild/content/eng/pack-sources.sh @@ -0,0 +1,96 @@ +#!/usr/bin/env bash + +### This script exports the sources of VMR (dotnet/dotnet) as an archive that can be used +### to build the .NET SDK. +### It expects you clone the dotnet/dotnet repo locally and check out the desired revision. +### +### USAGE: +### ./pack-sources.sh -o dotnet.tar.gz +### Options: +### -o, --output PATH +### Optional. Path or dir where the archive is created. +### Defaults to artifacts/packages/dotnet-[SHA].tar.gz + +source="${BASH_SOURCE[0]}" + +# resolve $source until the file is no longer a symlink +while [[ -h "$source" ]]; do + scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + source="$(readlink "$source")" + # if $source was a relative symlink, we need to resolve it relative to the path where the + # symlink file was located + [[ $source != /* ]] && source="$scriptroot/$source" +done +scriptroot="$( cd -P "$( dirname "$source" )" && pwd )" + +function print_help () { + sed -n '/^### /,/^$/p' "$source" | cut -b 5- +} + +GIT_ROOT=$(realpath "$scriptroot/../") + +output='' + +while [[ $# -gt 0 ]]; do + opt="$(echo "$1" | tr "[:upper:]" "[:lower:]")" + case "$opt" in + -o|--output) + output=$2 + shift + ;; + -h|--help) + print_help + exit 0 + ;; + *) + fail "Invalid argument: $1" + usage + exit 1 + ;; + esac + + shift +done + +revision=$(git -C "$GIT_ROOT" rev-parse HEAD) +filename="dotnet-$revision" + +if [[ -z "$output" ]]; then + output="$GIT_ROOT/artifacts/packages/$filename.tar.gz" +fi + +# If output is directory, use the default filename +if [[ -d "$output" ]]; then + output="$output/$filename.tar.gz" +fi + +set -e + +echo "Packing sources of $revision to $output.." +mkdir -p "$(dirname "$output")" +rm -f "$output" + +tmp="$GIT_ROOT/artifacts/tmp" +mkdir -p "$tmp" +config_file="$tmp/config" +HEAD_file="$tmp/HEAD" + +start_time=$(date +%s) + +# We need to had `.git/HEAD` and `.git/config` to the archive as the build expects those +echo $'[remote "origin"]\nurl="http://github.com/dotnet/dotnet"' > "$config_file" +echo "$revision" > "$HEAD_file" + +git -C "$GIT_ROOT" archive \ + -o "$output" \ + --prefix "$filename/.git/" \ + --add-file "$config_file" \ + --add-file "$HEAD_file" \ + --prefix "$filename/" \ + "$revision" "$GIT_ROOT" + +end_time=$(date +%s) +elapsed=$(( end_time - start_time )) + +duration=$(date -u +%H:%M:%S "-d@$elapsed") +echo "Archive created in $duration" diff --git a/src/SourceBuild/content/eng/tools/Directory.Build.props b/src/SourceBuild/content/eng/tools/Directory.Build.props index bffd69d62..17a556ec5 100644 --- a/src/SourceBuild/content/eng/tools/Directory.Build.props +++ b/src/SourceBuild/content/eng/tools/Directory.Build.props @@ -2,7 +2,7 @@ true diff --git a/src/SourceBuild/content/eng/tools/init-build.proj b/src/SourceBuild/content/eng/tools/init-build.proj index 1206c1b50..319333041 100644 --- a/src/SourceBuild/content/eng/tools/init-build.proj +++ b/src/SourceBuild/content/eng/tools/init-build.proj @@ -9,7 +9,6 @@ - @@ -23,7 +22,6 @@ - - - - - - - - - - - - - - - - $(IntermediatePath)PackageVersions.props - - - - - - -]]> - - - - - @@ -119,16 +86,6 @@ - - - - - - /// Source usage data JSON file. @@ -26,7 +26,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport public string DataFile { get; set; } /// - /// A set of "PackageVersions.props.pre.{repo}.xml" files. They are analyzed to find + /// A set of "PackageVersions.{repo}.Current.props" files. They are analyzed to find /// packages built during source-build, and which repo built them. This info is added to the /// report. New packages are associated to a repo by going through each PVP in ascending /// file modification order. @@ -114,7 +114,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport string id = usage.PackageIdentity.Id; string version = usage.PackageIdentity.Version.OriginalVersion; - string pvpIdent = WriteBuildOutputProps.GetPropertyName(id); + string pvpIdent = WritePackageVersionsProps.GetPropertyName(id, WritePackageVersionsProps.VersionPropertySuffix); var sourceBuildCreator = new StringBuilder(); foreach (RepoOutput output in sourceBuildRepoOutputs) @@ -200,7 +200,7 @@ namespace Microsoft.DotNet.SourceBuild.Tasks.UsageReport // Get the creation time element. ?.Element(snapshot.Xml .GetDefaultNamespace() - .GetName(WriteBuildOutputProps.CreationTimePropertyName)) + .GetName(WritePackageVersionsProps.CreationTimePropertyName)) ?.Value; if (string.IsNullOrEmpty(creationTime)) diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteBuildOutputProps.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteBuildOutputProps.cs deleted file mode 100644 index fb1d2fe4d..000000000 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteBuildOutputProps.cs +++ /dev/null @@ -1,142 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using NuGet.Packaging; -using NuGet.Packaging.Core; -using NuGet.Versioning; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; - -namespace Microsoft.DotNet.Build.Tasks -{ - public class WriteBuildOutputProps : Task - { - private static readonly Regex InvalidElementNameCharRegex = new Regex(@"(^|[^A-Za-z0-9])(?.)"); - - public const string CreationTimePropertyName = "BuildOutputPropsCreationTime"; - - [Required] - public ITaskItem[] NuGetPackages { get; set; } - - [Required] - public string OutputPath { get; set; } - - /// - /// Adds a second PropertyGroup to the output XML containing a property with the time of - /// creation in UTC DateTime Ticks. This can be used to track creation time in situations - /// where file metadata isn't reliable or preserved. - /// - public bool IncludeCreationTimeProperty { get; set; } - - /// - /// Properties to add to the build output props, which may not exist as nupkgs. - /// FOr example, this is used to pass the version of the CLI toolset archives. - /// - /// %(Identity): Package identity. - /// %(Version): Package version. - /// - public ITaskItem[] ExtraProperties { get; set; } - - /// - /// Additional assets to be added to the build output props. - /// i.e. /bin/obj/x64/Release/blobs/Toolset/3.0.100 - /// This parameter is the / portion only, and the asset - /// must be in a / folder. - /// - public string[] AdditionalAssetDirs { get; set; } - - public override bool Execute() - { - PackageIdentity[] latestPackages = NuGetPackages - .Select(item => - { - using (var reader = new PackageArchiveReader(item.GetMetadata("FullPath"))) - { - return reader.GetIdentity(); - } - }) - .GroupBy(identity => identity.Id) - .Select(g => g.OrderBy(id => id.Version).Last()) - .OrderBy(id => id.Id) - .ToArray(); - - var additionalAssets = (AdditionalAssetDirs ?? new string[0]) - .Where(Directory.Exists) - .Where(dir => Directory.GetDirectories(dir).Count() > 0) - .Select(dir => new { - Name = new DirectoryInfo(dir).Name + "Version", - Version = new DirectoryInfo(Directory.EnumerateDirectories(dir).OrderBy(s => s).Last()).Name - }).ToArray(); - - Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); - - using (var outStream = File.Open(OutputPath, FileMode.Create)) - using (var sw = new StreamWriter(outStream, new UTF8Encoding(false))) - { - sw.WriteLine(@""); - sw.WriteLine(@""); - sw.WriteLine(@" "); - foreach (PackageIdentity packageIdentity in latestPackages) - { - string propertyName = GetPropertyName(packageIdentity.Id); - sw.WriteLine($" <{propertyName}>{packageIdentity.Version}"); - - propertyName = GetAlternatePropertyName(packageIdentity.Id); - sw.WriteLine($" <{propertyName}>{packageIdentity.Version}"); - } - foreach (var extraProp in ExtraProperties ?? Enumerable.Empty()) - { - string propertyName = extraProp.GetMetadata("Identity"); - bool doNotOverwrite = false; - string overwriteCondition = string.Empty; - if (bool.TryParse(extraProp.GetMetadata("DoNotOverwrite"), out doNotOverwrite) && doNotOverwrite) - { - overwriteCondition = $" Condition=\"'$({propertyName})' == ''\""; - } - sw.WriteLine($" <{propertyName}{overwriteCondition}>{extraProp.GetMetadata("Version")}"); - } - foreach (var additionalAsset in additionalAssets) - { - sw.WriteLine($" <{additionalAsset.Name}>{additionalAsset.Version}"); - } - sw.WriteLine(@" "); - if (IncludeCreationTimeProperty) - { - sw.WriteLine(@" "); - sw.WriteLine($@" <{CreationTimePropertyName}>{DateTime.UtcNow.Ticks}"); - sw.WriteLine(@" "); - } - sw.WriteLine(@""); - } - - return true; - } - - public static string GetPropertyName(string id) - { - string formattedId = InvalidElementNameCharRegex.Replace( - id, - match => match.Groups?["FirstPartChar"].Value.ToUpperInvariant() - ?? string.Empty); - - return $"{formattedId}PackageVersion"; - } - - public static string GetAlternatePropertyName(string id) - { - string formattedId = InvalidElementNameCharRegex.Replace( - id, - match => match.Groups?["FirstPartChar"].Value.ToUpperInvariant() - ?? string.Empty); - - return $"{formattedId}Version"; - } - } -} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WritePackageVersionProps.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WritePackageVersionProps.cs new file mode 100644 index 000000000..bfc98e487 --- /dev/null +++ b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WritePackageVersionProps.cs @@ -0,0 +1,326 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using Microsoft.Build.Framework; +using Microsoft.Build.Utilities; +using NuGet.Packaging; +using NuGet.Packaging.Core; +using NuGet.Versioning; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Text.RegularExpressions; +using System.Xml; + +namespace Microsoft.DotNet.Build.Tasks +{ + public class VersionEntry + { + public string Name; + public string Version; + } + + /// + /// Creates a props file that is used as the input for a repo-level build. The props file + /// includes package version numbers that should be used by the repo build and additional special properties. + /// + /// There are two types of input props that can be written: + /// - Versions of union of all packages produced by the builds are added. (AllPackages) + /// - Only versions of packages that are listed as dependencies of a repo are added. (DependenciesOnly) + /// + /// The former represents the current way that source build works for most repos. The latter represents the desired + /// methodology (PVP Flow). PVP flow closely matches how the product is built in non-source-build mode. + /// + public class WritePackageVersionsProps : Microsoft.Build.Utilities.Task + { + private static readonly Regex InvalidElementNameCharRegex = new Regex(@"(^|[^A-Za-z0-9])(?.)"); + + public const string CreationTimePropertyName = "BuildOutputPropsCreationTime"; + public const string VersionPropertySuffix = "Version"; + private const string VersionPropertyAlternateSuffix = "PackageVersion"; + private const string PinnedAttributeName = "Pinned"; + private const string DependencyAttributeName = "Dependency"; + private const string NameAttributeName = "Name"; + + private const string AllPackagesVersionPropsFlowType = "AllPackages"; + private const string DependenciesOnlyVersionPropsFlowType = "DependenciesOnly"; + private const string DefaultVersionPropsFlowType = AllPackagesVersionPropsFlowType; + + /// + /// Set of input nuget package files to generate version properties for. + /// + [Required] + public ITaskItem[] NuGetPackages { get; set; } + + /// + /// File where the version properties should be written. + /// + [Required] + public string OutputPath { get; set; } + + /// + /// Adds a second PropertyGroup to the output XML containing a property with the time of + /// creation in UTC DateTime Ticks. This can be used to track creation time in situations + /// where file metadata isn't reliable or preserved. + /// + public bool IncludeCreationTimeProperty { get; set; } + + /// + /// Properties to add to the build output props, which may not exist as nupkgs. + /// FOr example, this is used to pass the version of the CLI toolset archives. + /// + /// %(Identity): Package identity. + /// %(Version): Package version. + /// + public ITaskItem[] ExtraProperties { get; set; } + + /// + /// Additional assets to be added to the build output props. + /// i.e. /bin/obj/x64/Release/blobs/Toolset/3.0.100 + /// This parameter is the / portion only, and the asset + /// must be in a / folder. + /// + public string[] AdditionalAssetDirs { get; set; } + + /// + /// Indicates which properties will be written into the Version props file. + /// If AllPackages (Default), all packages from previously built repos will be written. + /// If DependenciesOnly, then only those packages appearing as dependencies in + /// Version.Details.xml will show up. The VersionsDetails property must be set to a + /// valid Version.Details.xml path when DependenciesOnly is used. + /// + public string VersionPropsFlowType { get; set; } = DefaultVersionPropsFlowType; + + /// + /// If VersionPropsFlowType is set to DependenciesOnly, should be the path to the Version.Detail.xml file for the repo. + /// + public string VersionDetails { get; set; } + + /// + /// Retrieve the set of the dependencies from the repo's Version.Details.Xml file. + /// + /// Hash set of dependency names. + private HashSet GetDependences() + { + XmlDocument document = new XmlDocument(); + + try + { + document.Load(VersionDetails); + } + catch (Exception e) + { + Log.LogErrorFromException(e); + return null; + } + + HashSet dependencyNames = new HashSet(StringComparer.OrdinalIgnoreCase); + + // Load the nodes, filter those that are not pinned, and + XmlNodeList dependencyNodes = document.DocumentElement.SelectNodes($"//{DependencyAttributeName}"); + + foreach (XmlNode dependency in dependencyNodes) + { + if (dependency.NodeType == XmlNodeType.Comment || dependency.NodeType == XmlNodeType.Whitespace) + { + continue; + } + + bool isPinned = false; + XmlAttribute pinnedAttribute = dependency.Attributes[PinnedAttributeName]; + if (pinnedAttribute != null && !bool.TryParse(pinnedAttribute.Value, out isPinned)) + { + Log.LogError($"The '{PinnedAttributeName}' attribute is set but the value " + + $"'{pinnedAttribute.Value}' is not a valid boolean..."); + return null; + } + + if (isPinned) + { + continue; + } + + var name = dependency.Attributes[NameAttributeName]?.Value?.Trim(); + + if (string.IsNullOrEmpty(name)) + { + Log.LogError($"The '{NameAttributeName}' attribute must be specified."); + return null; + } + + dependencyNames.Add(name); + } + + return dependencyNames; + } + + /// + /// Filter a set of input dependencies to those that appear in + /// + /// Input set of entries + /// Set of dependencies + /// Set of that appears in + private IEnumerable FilterNonDependencies(IEnumerable input, HashSet dependencies) + { + return input.Where(entry => dependencies.Contains(entry.Name)); + } + + public override bool Execute() + { + if (VersionPropsFlowType != AllPackagesVersionPropsFlowType && + VersionPropsFlowType != DependenciesOnlyVersionPropsFlowType) + { + Log.LogError($"Valid version flow types are '{DependenciesOnlyVersionPropsFlowType}' and '{AllPackagesVersionPropsFlowType}'"); + return !Log.HasLoggedErrors; + } + + if (VersionPropsFlowType == DependenciesOnlyVersionPropsFlowType && (string.IsNullOrEmpty(VersionDetails) || !File.Exists(VersionDetails))) + { + Log.LogError($"When version flow type is DependenciesOnly, the VersionDetails task parameter must point to a valid path to the Version.Details.xml file for the repo. " + + "Provided file path '{VersionDetails}' does not exist."); + return !Log.HasLoggedErrors; + } + + // First, obtain version information from the packages and additional assets that + // are provided. + var latestPackages = NuGetPackages + .Select(item => + { + using (var reader = new PackageArchiveReader(item.GetMetadata("FullPath"))) + { + return reader.GetIdentity(); + } + }) + .GroupBy(identity => identity.Id) + .Select(g => g.OrderBy(id => id.Version).Last()) + .OrderBy(id => id.Id) + .Select(identity => new VersionEntry() + { + Name = identity.Id, + Version = identity.Version.ToString() + }); + + var additionalAssets = (AdditionalAssetDirs ?? new string[0]) + .Where(Directory.Exists) + .Where(dir => Directory.GetDirectories(dir).Count() > 0) + .Select(dir => new VersionEntry() + { + Name = new DirectoryInfo(dir).Name, + Version = new DirectoryInfo(Directory.EnumerateDirectories(dir).OrderBy(s => s).Last()).Name + }); + + var packageElementsToWrite = latestPackages; + var additionalAssetElementsToWrite = additionalAssets; + + // Then, if version flow type is "DependenciesOnly", filter those + // dependencies that do not appear in the version.details.xml file. + if (VersionPropsFlowType == DependenciesOnlyVersionPropsFlowType) + { + var dependencies = GetDependences(); + + if (Log.HasLoggedErrors) + { + return false; + } + + packageElementsToWrite = FilterNonDependencies(packageElementsToWrite, dependencies); + additionalAssetElementsToWrite = FilterNonDependencies(additionalAssetElementsToWrite, dependencies); + } + + Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); + + using (var outStream = File.Open(OutputPath, FileMode.Create)) + using (var sw = new StreamWriter(outStream, new UTF8Encoding(false))) + { + sw.WriteLine(@""); + sw.WriteLine(@""); + + WriteVersionEntries(sw, packageElementsToWrite, "packages"); + WriteExtraProperties(sw); + WriteVersionEntries(sw, additionalAssetElementsToWrite, "additional assets"); + + if (IncludeCreationTimeProperty) + { + sw.WriteLine(@" "); + sw.WriteLine($@" <{CreationTimePropertyName}>{DateTime.UtcNow.Ticks}"); + sw.WriteLine(@" "); + } + + sw.WriteLine(@""); + } + + return !Log.HasLoggedErrors; + } + + /// + /// Write properties specified in the "ExtraProperties task parameter + /// + /// Stream writer + private void WriteExtraProperties(StreamWriter sw) + { + if (ExtraProperties == null) + { + return; + } + + sw.WriteLine(@" "); + sw.WriteLine(@" "); + + foreach (var extraProp in ExtraProperties ?? Enumerable.Empty()) + { + string propertyName = extraProp.GetMetadata("Identity"); + bool doNotOverwrite = false; + string overwriteCondition = string.Empty; + if (bool.TryParse(extraProp.GetMetadata("DoNotOverwrite"), out doNotOverwrite) && doNotOverwrite) + { + overwriteCondition = $" Condition=\"'$({propertyName})' == ''\""; + } + sw.WriteLine($" <{propertyName}{overwriteCondition}>{extraProp.GetMetadata("Version")}"); + } + + sw.WriteLine(@" "); + } + + /// + /// Write properties for the version numbers required for this repo. + /// + /// Stream writer + /// Version entries + private void WriteVersionEntries(StreamWriter sw, IEnumerable entries, string entryType) + { + if (!entries.Any()) + { + return; + } + + sw.WriteLine($" "); + if (VersionPropsFlowType == DependenciesOnlyVersionPropsFlowType) + { + sw.WriteLine(@" "); + } + sw.WriteLine(@" "); + foreach (var package in entries) + { + string propertyName = GetPropertyName(package.Name, VersionPropertySuffix); + string alternatePropertyName = GetPropertyName(package.Name, VersionPropertyAlternateSuffix); + + sw.WriteLine($" <{propertyName}>{package.Version}"); + sw.WriteLine($" <{alternatePropertyName}>{package.Version}"); + } + sw.WriteLine(@" "); + } + + public static string GetPropertyName(string id, string suffix) + { + string formattedId = InvalidElementNameCharRegex.Replace( + id, + match => match.Groups?["FirstPartChar"].Value.ToUpperInvariant() + ?? string.Empty); + + return $"{formattedId}{suffix}"; + } + } +} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceAndVersionProps.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceAndVersionProps.cs deleted file mode 100644 index ee8a9ba7d..000000000 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceAndVersionProps.cs +++ /dev/null @@ -1,156 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using NuGet.Packaging; -using NuGet.Packaging.Core; -using NuGet.Versioning; -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Text; -using System.Text.RegularExpressions; - -namespace Microsoft.DotNet.Build.Tasks -{ - public class WriteRestoreSourceAndVersionProps : Task - { - private static readonly Regex InvalidElementNameCharRegex = new Regex(@"(^|[^A-Za-z0-9])(?.)"); - - public const string CreationTimePropertyName = "BuildOutputPropsCreationTime"; - - [Required] - public ITaskItem[] RestoreSources { get; set; } - - [Required] - public ITaskItem[] NuGetPackages { get; set; } - - [Required] - public string OutputPath { get; set; } - - /// - /// Adds a second PropertyGroup to the output XML containing a property with the time of - /// creation in UTC DateTime Ticks. This can be used to track creation time in situations - /// where file metadata isn't reliable or preserved. - /// - public bool IncludeCreationTimeProperty { get; set; } - - /// - /// Properties to add to the build output props, which may not exist as nupkgs. - /// FOr example, this is used to pass the version of the CLI toolset archives. - /// - /// %(Identity): Package identity. - /// %(Version): Package version. - /// - public ITaskItem[] ExtraProperties { get; set; } - - /// - /// Additional assets to be added to the build output props. - /// i.e. /bin/obj/x64/Release/blobs/Toolset/3.0.100 - /// This parameter is the / portion only, and the asset - /// must be in a / folder. - /// - public string[] AdditionalAssetDirs { get; set; } - - public override bool Execute() - { - PackageIdentity[] latestPackages = NuGetPackages - .Select(item => - { - using (var reader = new PackageArchiveReader(item.GetMetadata("FullPath"))) - { - return reader.GetIdentity(); - } - }) - .GroupBy(identity => identity.Id) - .Select(g => g.OrderBy(id => id.Version).Last()) - .OrderBy(id => id.Id) - .ToArray(); - - var additionalAssets = (AdditionalAssetDirs ?? new string[0]) - .Where(Directory.Exists) - .Where(dir => Directory.GetDirectories(dir).Count() > 0) - .Select(dir => new { - Name = new DirectoryInfo(dir).Name + "Version", - Version = new DirectoryInfo(Directory.EnumerateDirectories(dir).OrderBy(s => s).Last()).Name - }).ToArray(); - - Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); - - using (var outStream = File.Open(OutputPath, FileMode.Create)) - using (var sw = new StreamWriter(outStream, new UTF8Encoding(false))) - { - sw.WriteLine(@""); - sw.WriteLine(@""); - sw.WriteLine(@" "); - sw.WriteLine(@" "); - foreach (ITaskItem restoreSourceItem in RestoreSources) - { - sw.WriteLine($" {restoreSourceItem.ItemSpec};"); - } - sw.WriteLine(@" "); - sw.WriteLine(@" "); - - sw.WriteLine(@" "); - foreach (PackageIdentity packageIdentity in latestPackages) - { - string propertyName = GetPropertyName(packageIdentity.Id); - string shortPropertyName = GetShortPropertyName(packageIdentity.Id); - - sw.WriteLine($" <{propertyName}>{packageIdentity.Version}"); - sw.WriteLine($" <{shortPropertyName}>{packageIdentity.Version}"); - } - foreach (var extraProp in ExtraProperties ?? Enumerable.Empty()) - { - string propertyName = extraProp.GetMetadata("Identity"); - bool doNotOverwrite = false; - string overwriteCondition = string.Empty; - if (bool.TryParse(extraProp.GetMetadata("DoNotOverwrite"), out doNotOverwrite) && doNotOverwrite) - { - overwriteCondition = $" Condition=\"'$({propertyName})' == ''\""; - } - sw.WriteLine($" <{propertyName}{overwriteCondition}>{extraProp.GetMetadata("Version")}"); - } - foreach (var additionalAsset in additionalAssets) - { - sw.WriteLine($" <{additionalAsset.Name}>{additionalAsset.Version}"); - } - sw.WriteLine(@" "); - - if (IncludeCreationTimeProperty) - { - sw.WriteLine(@" "); - sw.WriteLine($@" <{CreationTimePropertyName}>{DateTime.UtcNow.Ticks}"); - sw.WriteLine(@" "); - } - - sw.WriteLine(@""); - } - - return true; - } - - public static string GetPropertyName(string id) - { - string formattedId = InvalidElementNameCharRegex.Replace( - id, - match => match.Groups?["FirstPartChar"].Value.ToUpperInvariant() - ?? string.Empty); - - return $"{formattedId}PackageVersion"; - } - - public static string GetShortPropertyName(string id) - { - string formattedId = InvalidElementNameCharRegex.Replace( - id, - match => match.Groups?["FirstPartChar"].Value.ToUpperInvariant() - ?? string.Empty); - - return $"{formattedId}Version"; - } - } -} diff --git a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceProps.cs b/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceProps.cs deleted file mode 100644 index 8e983531b..000000000 --- a/src/SourceBuild/content/eng/tools/tasks/Microsoft.DotNet.SourceBuild.Tasks.XPlat/WriteRestoreSourceProps.cs +++ /dev/null @@ -1,43 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using Microsoft.Build.Framework; -using Microsoft.Build.Utilities; -using System.IO; -using System.Text; - -namespace Microsoft.DotNet.Build.Tasks -{ - public class WriteRestoreSourceProps : Task - { - [Required] - public ITaskItem[] RestoreSources { get; set; } - - [Required] - public string OutputPath { get; set; } - - public override bool Execute() - { - Directory.CreateDirectory(Path.GetDirectoryName(OutputPath)); - - using (var outStream = File.Open(OutputPath, FileMode.Create)) - using (var sw = new StreamWriter(outStream, new UTF8Encoding(false))) - { - sw.WriteLine(@""); - sw.WriteLine(@""); - sw.WriteLine(@" "); - sw.WriteLine(@" "); - foreach (ITaskItem restoreSourceItem in RestoreSources) - { - sw.WriteLine($" {restoreSourceItem.ItemSpec};"); - } - sw.WriteLine(@" "); - sw.WriteLine(@" "); - sw.WriteLine(@""); - } - - return true; - } - } -} diff --git a/src/SourceBuild/content/prep.sh b/src/SourceBuild/content/prep.sh index 4fcb7e1bb..0e7882806 100755 --- a/src/SourceBuild/content/prep.sh +++ b/src/SourceBuild/content/prep.sh @@ -7,7 +7,7 @@ SCRIPT_ROOT="$(cd -P "$( dirname "$0" )" && pwd)" usage() { echo "usage: $0 [options]" echo "" - echo " Prepares a tarball to be built by downloading Private.SourceBuilt.Artifacts.*.tar.gz and" + echo " Prepares the environment to be built by downloading Private.SourceBuilt.Artifacts.*.tar.gz and" echo " installing the version of dotnet referenced in global.json" echo "options:" echo " --bootstrap Build a bootstrap version of previously source-built packages archive." diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.props b/src/SourceBuild/content/repo-projects/Directory.Build.props index 4b4cacd31..337b9df49 100644 --- a/src/SourceBuild/content/repo-projects/Directory.Build.props +++ b/src/SourceBuild/content/repo-projects/Directory.Build.props @@ -22,6 +22,12 @@ Repo specific semaphore path for incremental build --> $(CompletedSemaphorePath)$(RepositoryName)/ + + + $(IntermediatePath)PackageVersions.$(RepositoryName).Current.props + $(IntermediatePath)PackageVersions.$(RepositoryName).Previous.props + $(IntermediatePath)PackageVersions.$(RepositoryName).props + AllPackages @@ -39,7 +45,7 @@ - '$(RepositoryName) in tarball' + '$(RepositoryName)' @@ -68,7 +74,6 @@ - diff --git a/src/SourceBuild/content/repo-projects/Directory.Build.targets b/src/SourceBuild/content/repo-projects/Directory.Build.targets index e663f8937..d10278261 100644 --- a/src/SourceBuild/content/repo-projects/Directory.Build.targets +++ b/src/SourceBuild/content/repo-projects/Directory.Build.targets @@ -13,14 +13,13 @@ - + - - + - $(RepoApiArgs) /p:DotNetRestoreSourcePropsPath=$(RestoreSourcePropsPath) $(RepoApiArgs) /p:DotNetBuildOffline=true @@ -150,7 +148,7 @@ NewText="_InitializeBuildToolFramework="%24{_OverrideArcadeInitializeBuildToolFramework-netcoreapp3.1}"" /> + the build process. https://github.com/dotnet/source-build/issues/2307 --> @@ -170,10 +168,8 @@ reference-packages source-built ExtraSources - dotnet5-internal-transport $(PrebuiltNuGetSourceName);$(PreviouslySourceBuiltNuGetSourceName);$(ReferencePackagesNuGetSourceName);$(SourceBuiltNuGetSourceName) $(SourceBuildSources);$(ExtraSourcesNuGetSourceName) - $(SourceBuildSources);$(DotNet5InternalTransportNuGetSourceName) @@ -210,78 +206,11 @@ SourcePath="$(ExtraRestoreSourcePath)" Condition="'$(ExtraRestoreSourcePath)' != ''" /> - - - - - - - - - - - - - - - - - - - - - - - - - @@ -303,59 +232,67 @@ - + + Outputs="$(RepoCompletedSemaphorePath)CreateBuildInputProps.complete"> + + + <_CurrentSourceBuiltPackages Include="$(SourceBuiltPackagesPath)*.nupkg" + Exclude="$(SourceBuiltPackagesPath)*.symbols.nupkg" /> + <_PreviouslyBuiltSourceBuiltPackages Include="$(PrebuiltSourceBuiltPackagesPath)*.nupkg" /> + + <_CurrentAdditionalAssetDirs Include="$(SourceBuiltToolsetDir)" Condition="Exists('$(SourceBuiltToolsetDir)')" /> + + + + - <_PackageVersionPropsBackupPath>$(PackageVersionPropsPath).pre.$(RepositoryName).xml + <_VersionDetailsXml Condition="'$(PackageVersionPropsFlowType)' == 'DependenciesOnly'">$(ProjectDirectory)/eng/Version.Details.xml - - - <_AdditionalAssetDirs Include="$(SourceBuiltToolsetDir)" Condition="Exists('$(SourceBuiltToolsetDir)')" /> - - - + - + + - + + - - - + + + + + + + + +]]> + + - - + - - - <_DotNetRestoreSources Include="$(ExtraRestoreSourcePath)" Condition="'$(ExtraRestoreSourcePath)' != ''"/> - <_DotNetRestoreSources Include="$(SourceBuiltPackagesPath)" /> - <_DotNetRestoreSources Include="$(ReferencePackagesDir)"/> - <_DotNetRestoreSources Include="$(PrebuiltPackagesPath)"/> - <_DotNetRestoreSources Include="$(PrebuiltSourceBuiltPackagesPath)"/> - - <_AdditionalAssetDirs Include="$(SourceBuiltToolsetDir)" Condition="Exists('$(SourceBuiltToolsetDir)')" /> - - + - + <_PackagesNotCreatedReason Include="^ There may have been a silent failure in the submodule build. To confirm, check the build log file for undetected errors that may have prevented package creation: $(RepoConsoleLogFile)" /> <_PackagesNotCreatedReason Include="^ This error might be a false positive if $(RepositoryName) intentionally builds no nuget packages. If so, set the SkipEnsurePackagesCreated property to true in $(MSBuildProjectFullPath)" /> - <_PackagesNotCreatedReason Include="^ The 'bin' directory might be dirty from a previous build and the package files already existed. If so, perform a clean build, or check which packages were already in 'bin' by opening $(_PackageVersionPropsBackupPath)" /> + <_PackagesNotCreatedReason Include="^ The 'bin' directory might be dirty from a previous build and the package files already existed. If so, perform a clean build, or check which packages were already in 'bin' by opening $(CurrentSourceBuiltPackageVersionPropsPath)" /> <_PackagesNotCreatedReason Include="^ The packages may have been written to an unexpected directory. For example, some repos used bin/ and changed to artifacts/ to match Arcade. Check PackagesOutput in $(MSBuildProjectFullPath) (currently '$(PackagesOutput)')" /> @@ -702,7 +639,7 @@ - + diff --git a/src/SourceBuild/content/repo-projects/aspnetcore.proj b/src/SourceBuild/content/repo-projects/aspnetcore.proj index 1849b9582..a19b4ec2e 100644 --- a/src/SourceBuild/content/repo-projects/aspnetcore.proj +++ b/src/SourceBuild/content/repo-projects/aspnetcore.proj @@ -6,6 +6,8 @@ osx-$(Platform) freebsd-$(Platform) win-$(Platform) + <_portableRidOverridden Condition="'$(TargetRid)' != '$(OverrideTargetRid)'">true + <_portableRidOverridden Condition="'$(TargetRid)' == '$(OverrideTargetRid)'">false $(StandardSourceBuildArgs.Replace('--publish', '')) @@ -14,7 +16,7 @@ $(BuildCommandArgs) --no-build-repo-tasks $(BuildCommandArgs) --no-build-nodejs $(BuildCommandArgs) /p:PublishCompressedFilesPathPrefix=$(SourceBuiltAspNetCoreRuntime) - $(BuildCommandArgs) /p:PortableBuild=false /p:TargetRuntimeIdentifier=$(OverrideTargetRid) + $(BuildCommandArgs) /p:PortableBuild=$(_portableRidOverridden) /p:TargetRuntimeIdentifier=$(OverrideTargetRid) $(BuildCommandArgs) /p:MicrosoftNetFrameworkReferenceAssembliesVersion=1.0.0 diff --git a/src/SourceBuild/content/repo-projects/installer.proj b/src/SourceBuild/content/repo-projects/installer.proj index 42ef91233..d41b97b78 100644 --- a/src/SourceBuild/content/repo-projects/installer.proj +++ b/src/SourceBuild/content/repo-projects/installer.proj @@ -49,9 +49,6 @@ $(ProjectDirectory)NuGet.config - - true - CS9057 diff --git a/src/SourceBuild/content/repo-projects/known-good.proj b/src/SourceBuild/content/repo-projects/known-good.proj index 989d6e20d..2d2226637 100644 --- a/src/SourceBuild/content/repo-projects/known-good.proj +++ b/src/SourceBuild/content/repo-projects/known-good.proj @@ -53,8 +53,8 @@ - + diff --git a/src/SourceBuild/content/repo-projects/msbuild.proj b/src/SourceBuild/content/repo-projects/msbuild.proj index 0504e9702..58f840009 100644 --- a/src/SourceBuild/content/repo-projects/msbuild.proj +++ b/src/SourceBuild/content/repo-projects/msbuild.proj @@ -39,7 +39,7 @@ The CentralPackageVersions SDK isn't actually source-built. We get it as a text-only prebuilt, but the NuGet resolver seems flaky so we're using our resolver instead. We only have access to - the nupkg ahead of time when building a tarball, so only enable this workaround then. + the nupkg ahead of time when building, so only enable this workaround then. --> diff --git a/src/SourceBuild/content/repo-projects/sdk.proj b/src/SourceBuild/content/repo-projects/sdk.proj index 5d0fe449d..773d7301a 100644 --- a/src/SourceBuild/content/repo-projects/sdk.proj +++ b/src/SourceBuild/content/repo-projects/sdk.proj @@ -10,9 +10,6 @@ true $(BuildCommandArgs) -v $(LogVerbosity) - - $(BuildCommandArgs) /p:PB_PackageVersionPropsUrl=file:%2F%2F$(PackageVersionPropsPath) - $(StandardSourceBuildCommand) $(BuildCommandArgs) diff --git a/src/SourceBuild/content/repo-projects/templating.proj b/src/SourceBuild/content/repo-projects/templating.proj index 8c41dc974..ed0971d42 100644 --- a/src/SourceBuild/content/repo-projects/templating.proj +++ b/src/SourceBuild/content/repo-projects/templating.proj @@ -14,8 +14,7 @@ - - + diff --git a/src/VirtualMonoRepo/source-mappings.json b/src/VirtualMonoRepo/source-mappings.json index 006b53c47..95c8f5f00 100644 --- a/src/VirtualMonoRepo/source-mappings.json +++ b/src/VirtualMonoRepo/source-mappings.json @@ -108,7 +108,10 @@ { "name": "nuget-client", "defaultRemote": "https://github.com/NuGet/NuGet.Client", - "defaultRef": "dev" + "defaultRef": "dev", + "exclude":[ + "src/NuGet.Clients/NuGet.VisualStudio.Client" + ] }, { "name": "razor", diff --git a/src/redist/targets/GenerateBundledVersions.targets b/src/redist/targets/GenerateBundledVersions.targets index ef365ffaf..5896b95a9 100644 --- a/src/redist/targets/GenerateBundledVersions.targets +++ b/src/redist/targets/GenerateBundledVersions.targets @@ -68,10 +68,10 @@ 30 - 31 + 32 17 - 11 - 0 + 13 + 2 @@ -515,7 +515,7 @@ Copyright (c) .NET Foundation. All rights reserved. $(LayoutDirectory)MSBuildExtensions--> + + + Condition=" '$(GitCommitCount)' == '' AND '$(_PatchNumber)' == '' "> - - $(GitCommitCount.PadLeft(6,'0'))