[ArPow] Add runtime repo to tarball (#11506)

* Add runtime repo / update runtime project * Update runtime-portable logs and properties * Create runtime-portable git-info * Fix unpack of IntermediatePackages * Re-enable references to runtime * Pass TargetRid and NonPortable to runtime build * Only build allconfigurations in non-portable build * Add SDK patch to disable new CA warning * Increase built tarball timeout * Update prebuilt version
2021-08-18 12:18:01 -05:00 · 2021-08-18 12:18:01 -05:00 · 6bd49343d9
commit 6bd49343d9
parent 7c4b4e10e2
15 changed files with 105 additions and 33 deletions
--- a/eng/Versions.props
+++ b/eng/Versions.props
@ -159,7 +159,7 @@
    <MicrosoftBuildFrameworkVersion>15.7.179</MicrosoftBuildFrameworkVersion>
    <MicrosoftBuildUtilitiesCoreVersion>15.7.179</MicrosoftBuildUtilitiesCoreVersion>
    <PrivateSourceBuiltArtifactsPackageVersion>0.1.0-6.0.100-bootstrap.11</PrivateSourceBuiltArtifactsPackageVersion>
-    <PrivateSourceBuiltPrebuiltsPackageVersion>0.1.0-6.0.100-6</PrivateSourceBuiltPrebuiltsPackageVersion>
+    <PrivateSourceBuiltPrebuiltsPackageVersion>0.1.0-6.0.100-10</PrivateSourceBuiltPrebuiltsPackageVersion>
  </PropertyGroup>
  <!-- Workload manifest package versions -->
  <PropertyGroup>
--- a/src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml
+++ b/src/SourceBuild/Arcade/eng/common/templates/job/source-build-run-tarball-build.yml
@ -25,7 +25,7 @@ jobs:
      Fedora33-Offline:
        _runOnline: false
        _Container: mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-33-20210222183538-031e7d2
-  timeoutInMinutes: 120
+  timeoutInMinutes: 180
  variables:
    _TarballDir: $(Build.StagingDirectory)/tarball
  workspace:
--- a/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets
+++ b/src/SourceBuild/Arcade/tools/SourceBuildArcadeTarball.targets
@ -177,6 +177,10 @@
      <EngCommonContent Include="$(RepoRoot)eng/common/**/*" />
    </ItemGroup>

+    <Copy
+      SourceFiles="$(TarballGitInfoDir)runtime.props"
+      DestinationFiles="$(TarballGitInfoDir)runtime-portable.props" />
+
    <Copy
      SourceFiles="@(TarballContent)"
      DestinationFiles="@(TarballContent->'$(TarballRootDir)%(RecursiveDir)%(Filename)%(Extension)')" />
--- a/src/SourceBuild/tarball/content/eng/Versions.props
+++ b/src/SourceBuild/tarball/content/eng/Versions.props
@ -23,6 +23,6 @@
  <PropertyGroup>
    <PrivateSourceBuiltArtifactsPackageVersion>0.1.0-6.0.100-bootstrap.11</PrivateSourceBuiltArtifactsPackageVersion>
    <PrivateSourceBuiltPrebuiltsPackageVersionPrefix>0.1.0-6.0.100-</PrivateSourceBuiltPrebuiltsPackageVersionPrefix>
-    <PrivateSourceBuiltPrebuiltsPackageVersionSuffix>6</PrivateSourceBuiltPrebuiltsPackageVersionSuffix>
+    <PrivateSourceBuiltPrebuiltsPackageVersionSuffix>10</PrivateSourceBuiltPrebuiltsPackageVersionSuffix>
  </PropertyGroup>
 </Project>
--- a/src/SourceBuild/tarball/content/patches/runtime/0001-Conditionally-build-allconfigurations.patch
+++ b/src/SourceBuild/tarball/content/patches/runtime/0001-Conditionally-build-allconfigurations.patch
@ -0,0 +1,27 @@
+From c1eaa569a1ebe3aa688ca251fcbcb42ee086cf27 Mon Sep 17 00:00:00 2001
+From: dseefeld <dseefeld@microsoft.com>
+Date: Wed, 18 Aug 2021 12:20:24 +0000
+Subject: [PATCH] Conditionally build allconfigurations
+
+When building portable, only a subset of runtime needs to be
+built.  allconfigurations is only needed in a non-portable build.
+---
+ eng/SourceBuild.props | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/eng/SourceBuild.props b/eng/SourceBuild.props
+index 7dfc80d41fd..1e3bd504931 100644
+--- a/eng/SourceBuild.props
+++ b/eng/SourceBuild.props
+@@ -37,7 +37,7 @@
+       <InnerBuildArgs>$(InnerBuildArgs) --arch $(TargetRidPlatform)</InnerBuildArgs>
+       <InnerBuildArgs>$(InnerBuildArgs) --configuration $(Configuration)</InnerBuildArgs>
+       <InnerBuildArgs>$(InnerBuildArgs) --ci</InnerBuildArgs>
+-      <InnerBuildArgs>$(InnerBuildArgs) --allconfigurations</InnerBuildArgs>
+      <InnerBuildArgs  Condition="'$(SourceBuildNonPortable)' == 'true'">$(InnerBuildArgs) --allconfigurations</InnerBuildArgs>
+       <InnerBuildArgs>$(InnerBuildArgs) --verbosity $(LogVerbosity)</InnerBuildArgs>
+       <InnerBuildArgs>$(InnerBuildArgs) --nodereuse false</InnerBuildArgs>
+       <InnerBuildArgs>$(InnerBuildArgs) --warnAsError false</InnerBuildArgs>
+-- 
+2.31.1
+
--- a/src/SourceBuild/tarball/content/patches/sdk/0004-Disable-CA1416-for-ProcessExtensions.patch
+++ b/src/SourceBuild/tarball/content/patches/sdk/0004-Disable-CA1416-for-ProcessExtensions.patch
@ -0,0 +1,42 @@
+From 3e0c6887921670b5d9e312b121791307516a4ce2 Mon Sep 17 00:00:00 2001
+From: dseefeld <dseefeld@microsoft.com>
+Date: Wed, 18 Aug 2021 11:41:59 +0000
+Subject: [PATCH] Disable CA1416 for ProcessExtensions
+
+Since source-build loads an updated version of code analysis, this
+warning is showing up here.  Disable warning, since code is meant
+to be windows only.
+
+Eliminates errors like:
+    /src/tb-installer/src/sdk.5fab7585de482237f470634489b969a1bd03e063/artifacts/source-build/self/src/src/Cli/Microsoft.DotNet.Cli.Utils/Extensions/ProcessExtensions.cs(38,20): error CA1416: This call site is reachable on all platforms. 'ManagementObjectCollection.ManagementObjectEnumerator.MoveNext()' is only supported on: 'windows'. [/src/tb-installer/src/sdk.5fab7585de482237f470634489b969a1bd03e063/artifacts/source-build/self/src/src/Cli/Microsoft.DotNet.Cli.Utils/Microsoft.DotNet.Cli.Utils.csproj]
+    /src/tb-installer/src/sdk.5fab7585de482237f470634489b969a1bd03e063/artifacts/source-build/self/src/src/Cli/Microsoft.DotNet.Cli.Utils/Extensions/ProcessExtensions.cs(35,55): error CA1416: This call site is reachable on all platforms. 'ManagementObjectSearcher.Get()' is only supported on: 'windows'. [/src/tb-installer/src/sdk.5fab7585de482237f470634489b969a1bd03e063/artifacts/source-build/self/src/src/Cli/Microsoft.DotNet.Cli.Utils/Microsoft.DotNet.Cli.Utils.csproj]
+    /src/tb-installer/src/sdk.5fab7585de482237f470634489b969a1bd03e063/artifacts/source-build/self/src/src/Cli/Microsoft.DotNet.Cli.Utils/Extensions/ProcessExtensions.cs(36,80): error CA1416: This call site is reachable on all platforms. 'ManagementObjectCollection.GetEnumerator()' is only supported on: 'windows'. [/src/tb-installer/src/sdk.5fab7585de482237f470634489b969a1bd03e063/artifacts/source-build/self/src/src/Cli/Microsoft.DotNet.Cli.Utils/Microsoft.DotNet.Cli.Utils.csproj]
+    /src/tb-installer/src/sdk.5fab7585de482237f470634489b969a1bd03e063/artifacts/source-build/self/src/src/Cli/Microsoft.DotNet.Cli.Utils/Extensions/ProcessExtensions.cs(38,60): error CA1416: This call site is reachable on all platforms. 'ManagementObjectCollection.ManagementObjectEnumerator.Current' is only supported on: 'windows'. [/src/tb-installer/src/sdk.5fab7585de482237f470634489b969a1bd03e063/artifacts/source-build/self/src/src/Cli/Microsoft.DotNet.Cli.Utils/Microsoft.DotNet.Cli.Utils.csproj]
+    /src/tb-installer/src/sdk.5fab7585de482237f470634489b969a1bd03e063/artifacts/source-build/self/src/src/Cli/Microsoft.DotNet.Cli.Utils/Extensions/ProcessExtensions.cs(34,49): error CA1416: This call site is reachable on all platforms. 'ManagementObjectSearcher' is only supported on: 'windows'. [/src/tb-installer/src/sdk.5fab7585de482237f470634489b969a1bd03e063/artifacts/source-build/self/src/src/Cli/Microsoft.DotNet.Cli.Utils/Microsoft.DotNet.Cli.Utils.csproj]
+    /src/tb-installer/src/sdk.5fab7585de482237f470634489b969a1bd03e063/artifacts/source-build/self/src/src/Cli/Microsoft.DotNet.Cli.Utils/Extensions/ProcessExtensions.cs(38,60): error CA1416: This call site is reachable on all platforms. 'ManagementBaseObject.GetPropertyValue(string)' is only supported on: 'windows'. [/src/tb-installer/src/sdk.5fab7585de482237f470634489b969a1bd03e063/artifacts/source-build/self/src/src/Cli/Microsoft.DotNet.Cli.Utils/Microsoft.DotNet.Cli.Utils.csproj]
+---
+ .../Microsoft.DotNet.Cli.Utils/Extensions/ProcessExtensions.cs  | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/src/Cli/Microsoft.DotNet.Cli.Utils/Extensions/ProcessExtensions.cs b/src/Cli/Microsoft.DotNet.Cli.Utils/Extensions/ProcessExtensions.cs
+index a9e9880c7..82519dc63 100644
+--- a/src/Cli/Microsoft.DotNet.Cli.Utils/Extensions/ProcessExtensions.cs
+++ b/src/Cli/Microsoft.DotNet.Cli.Utils/Extensions/ProcessExtensions.cs
+@@ -12,6 +12,7 @@ namespace Microsoft.DotNet.Cli.Utils
+     /// </summary>
+     public static class ProcessExtensions
+     {
+#pragma warning disable CA1416
+         /// <summary>
+         /// Returns the parent process of this process by querying the Win32_Process class.
+         /// </summary>
+@@ -37,5 +38,6 @@ public static int GetParentProcessId(this Process process)
+ 
+             return enumerator.MoveNext() ? Convert.ToInt32(enumerator.Current.GetPropertyValue("ParentProcessId")) : -1;
+         }
+#pragma warning restore CA1416
+     }
+ }
+-- 
+2.31.1
+
--- a/src/SourceBuild/tarball/content/repos/Directory.Build.targets
+++ b/src/SourceBuild/tarball/content/repos/Directory.Build.targets
@ -496,17 +496,11 @@

    <ZipFileExtractToDirectory Condition="'@(_BuiltIntermediatePackages)' != ''"
                            SourceArchive="%(_BuiltIntermediatePackages.Identity)"
-                            DestinationDirectory="$(SourceBuiltPackagesPath)extractArtifacts/"
+                            DestinationDirectory="$(SourceBuiltPackagesPath)extractArtifacts/%(_BuiltIntermediatePackages.FileName)/"
                            OverwriteDestination="true" />

    <ItemGroup Condition="'@(_BuiltIntermediatePackages)' != ''">
-      <SourceBuiltNupkgFiles Include="$(SourceBuiltPackagesPath)extractArtifacts/**/*.nupkg" />
-      <!-- don't copy some packages that we specifically want to override for now due to bad reference package generation                                                  -->
-      <!-- relevant issues: https://github.com/dotnet/runtime/issues/44646, https://github.com/dotnet/runtime/issues/45183, https://github.com/dotnet/runtime/issues/45417 -->
-      <!-- these should be fixed for 6.0 proper                                                                                                                            -->
-      <PrebuiltPackagesToUse Include="$(PrebuiltPackagesPath)/*.nupkg" />
-      <SourceBuiltNupkgFilesToSkip Include="@(PrebuiltPackagesToUse -> '$(SourceBuiltPackagesPath)extractArtifacts/artifacts/%(Filename)%(Extension)')" />
-      <SourceBuiltNupkgFiles Condition="$([System.String]::Copy(%(_BuiltIntermediatePackages.Identity)).Contains('source-build-reference-packages'))" Remove="@(SourceBuiltNupkgFilesToSkip)" />
+      <SourceBuiltNupkgFiles Include="$(SourceBuiltPackagesPath)extractArtifacts/**/artifacts/*.*" />
    </ItemGroup>

    <Copy
--- a/src/SourceBuild/tarball/content/repos/aspnetcore.proj
+++ b/src/SourceBuild/tarball/content/repos/aspnetcore.proj
@ -26,8 +26,7 @@
  <ItemGroup>
    <RepositoryReference Include="arcade" />
    <RepositoryReference Include="source-build" />
-    <!-- TODO:  Enable with https://github.com/dotnet/source-build/issues/2274
-    <RepositoryReference Include="runtime" /> -->
+    <RepositoryReference Include="runtime" />
    <RepositoryReference Include="msbuild" />
    <RepositoryReference Include="roslyn" />
    <!-- TODO:  https://github.com/dotnet/source-build/issues/2319
--- a/src/SourceBuild/tarball/content/repos/known-good.proj
+++ b/src/SourceBuild/tarball/content/repos/known-good.proj
@ -45,6 +45,7 @@
        <RepositoryReference Include="linker" />
        <RepositoryReference Include="msbuild" />
        <RepositoryReference Include="runtime-portable" />
+        <RepositoryReference Include="runtime" />
        <RepositoryReference Include="templating" />

        <!-- Tier 4 -->
--- a/src/SourceBuild/tarball/content/repos/msbuild.proj
+++ b/src/SourceBuild/tarball/content/repos/msbuild.proj
@ -30,10 +30,8 @@

  <ItemGroup>
    <RepositoryReference Include="arcade" />
-    <!-- TODO:  Enable with https://github.com/dotnet/source-build/issues/2274
-    <RepositoryReference Include="runtime" /> -->
-    <!-- TODO:  Enable with https://github.com/dotnet/source-build/issues/2272 
-    <RepositoryReference Include="roslyn" /> -->
+    <RepositoryReference Include="runtime" />
+    <RepositoryReference Include="roslyn" />
  </ItemGroup>

  <!--
--- a/src/SourceBuild/tarball/content/repos/runtime-portable.proj
+++ b/src/SourceBuild/tarball/content/repos/runtime-portable.proj
@ -1,5 +1,9 @@
 <Project>
  <Import Project="runtime.common.props"/>
+  
+  <PropertyGroup>
+    <SourceDirectory>runtime</SourceDirectory>
+  </PropertyGroup>

  <!-- Repository References -->
  <ItemGroup>
@ -19,8 +23,9 @@
      <PortableObjDir>$(ProjectDirectory)artifacts/portableObj</PortableObjDir>
    </PropertyGroup>
    <ItemGroup>
-      <PortableLogFilesToCopy Include="$(ProjectDirectory)artifacts/log/**/*" />
-      <PortableObjFilesToCopy Include="$(ProjectDirectory)artifacts/obj/**/project.assets.json" />
+      <PortableLogFilesToCopy Include="$(ProjectDirectory)artifacts/**/*.log" />
+      <PortableLogFilesToCopy Include="$(ProjectDirectory)artifacts/**/*.binlog" />
+      <PortableObjFilesToCopy Include="$(ProjectDirectory)artifacts/**/project.assets.json" />
    </ItemGroup>
    <MakeDir Directories="$(PortableLogDir)" Condition="Exists('$(ProjectDirectory)artifacts')"/>
    <MakeDir Directories="$(PortableObjDir)" Condition="Exists('$(ProjectDirectory)artifacts')"/>
--- a/src/SourceBuild/tarball/content/repos/runtime.common.props
+++ b/src/SourceBuild/tarball/content/repos/runtime.common.props
@ -1,6 +1,5 @@
 <Project>
  <PropertyGroup>
-    <RepositoryName>runtime</RepositoryName>
    <SourceDirectory>runtime</SourceDirectory>
  </PropertyGroup>

--- a/src/SourceBuild/tarball/content/repos/runtime.proj
+++ b/src/SourceBuild/tarball/content/repos/runtime.proj
@ -1,22 +1,27 @@
 <Project>
  <Import Project="runtime.common.props"/>

+  <PropertyGroup>
+    <OverrideTargetRid>$(TargetRid)</OverrideTargetRid>
+    <OverrideTargetRid Condition="'$(TargetOS)' == 'OSX'">osx-x64</OverrideTargetRid>
+    <OverrideTargetRid Condition="'$(TargetOS)' == 'FreeBSD'">freebsd-x64</OverrideTargetRid>
+    <OverrideTargetRid Condition="'$(TargetOS)' == 'Windows_NT'">win-x64</OverrideTargetRid>
+
+    <BuildCommandArgs>$(StandardSourceBuildArgs)</BuildCommandArgs>
+    <BuildCommandArgs>$(BuildCommandArgs) /p:TargetRid=$(OverrideTargetRid)</BuildCommandArgs>
+    <BuildCommandArgs>$(BuildCommandArgs) /p:SourceBuildNonPortable=true</BuildCommandArgs>
+    <BuildCommand>$(StandardSourceBuildCommand) $(BuildCommandArgs)</BuildCommand>
+  </PropertyGroup>
+
  <!-- Repository References -->
  <ItemGroup>
      <RepositoryReference Include="arcade" />
      <RepositoryReference Include="symreader" />
      <RepositoryReference Include="linker" />
-      <RepositoryReference Include="newtonsoft-json" />
-      <RepositoryReference Include="newtonsoft-json901" />
+      <RepositoryReference Include="source-build" />
      <RepositoryReference Include="roslyn" />
-      <RepositoryReference Include="runtime-portable" Condition="'$(TargetOS)' == 'Linux' and '$(BuildPortableRuntime)' == 'true'" />
+      <RepositoryReference Include="runtime-portable" />
  </ItemGroup>

-    <!-- Additional Targets -->
-  <Target Name="InstallJustBuiltRuntime" AfterTargets="RemoveBuiltPackagesFromCache">
-    <!-- Install the runtime that was just built to be used by downstream repos, namely, aspnetcore -->
-    <Exec Command="tar -xvf $(SourceBuiltAssetsDir)dotnet-runtime-$(runtimeOutputPackageVersion)-$(OverrideTargetRid).tar.gz -C $(DotNetCliToolDir)" />
-  </Target>
-
  <Import Project="runtime.common.targets" />
 </Project>
--- a/src/SourceBuild/tarball/content/repos/sdk.proj
+++ b/src/SourceBuild/tarball/content/repos/sdk.proj
@ -35,8 +35,7 @@
  <ItemGroup>
    <RepositoryReference Include="arcade" />
    <RepositoryReference Include="xliff-tasks" />
-    <!-- TODO:  Enable with https://github.com/dotnet/source-build/issues/2274
-    <RepositoryReference Include="runtime" /> -->
+    <RepositoryReference Include="runtime" />
    <RepositoryReference Include="msbuild" />
    <RepositoryReference Include="source-build" />
    <!-- TODO:  https://github.com/dotnet/source-build/issues/2279
--- a/src/SourceBuild/tarball/content/repos/vstest.proj
+++ b/src/SourceBuild/tarball/content/repos/vstest.proj
@ -13,8 +13,7 @@

  <ItemGroup>
    <RepositoryReference Include="diagnostics" />
-    <!-- TODO:  Enable with https://github.com/dotnet/source-build/issues/2275
-    <RepositoryReference Include="runtime" /> -->
+    <RepositoryReference Include="runtime" />
    <RepositoryReference Include="source-build" />
  </ItemGroup>