Cleanup top level source-build directories (#15126)

* Rename repos dir to repo-projects

* Move keys dir into prereqs

* Refactor Tools and tools-local dirs

* Refactor ArcadeOverrides dir

* Update .gitignore

src/SourceBuild/tarball/content/.gitignore

@@ -1,8 +1,10 @@
 /.dotnet
-/Tools
 /artifacts
 /packages
-/tools-local/**/bin
-/tools-local/**/obj
+/eng/tools/**/bin
+/eng/tools/**/obj
 /src/linker/src/ILLink.Tasks/ILLink.Tasks.nuspec
 /src/nuget-client/NuGet.config
+/test/Microsoft.DotNet.SourceBuild.SmokeTests/bin
+/test/Microsoft.DotNet.SourceBuild.SmokeTests/obj
+/test/Microsoft.DotNet.SourceBuild.SmokeTests/TestResults

src/SourceBuild/tarball/content/Directory.Build.props

@@ -33,7 +33,7 @@
   <PropertyGroup>
     <ProjectDir>$(MSBuildThisFileDirectory)</ProjectDir>
     <TargetsDir>$(ProjectDir)targets/</TargetsDir>
-    <KeysDir>$(ProjectDir)keys/</KeysDir>
+    <KeysDir>$(ProjectDir)prereqs/keys/</KeysDir>
     <DotNetCliToolDir Condition="'$(DotNetCliToolDir)' == '' and '$(CustomDotNetSdkDir)' != ''">$([MSBuild]::EnsureTrailingSlash('$(CustomDotNetSdkDir)'))</DotNetCliToolDir>
     <DotNetCliToolDir Condition="'$(DotNetCliToolDir)' == '' and '$(DOTNET_INSTALL_DIR)' != ''">$([MSBuild]::NormalizeDirectory('$(DOTNET_INSTALL_DIR)'))</DotNetCliToolDir>
     <DotNetCliToolDir Condition="'$(DotNetCliToolDir)' == ''">$(ProjectDir).dotnet/</DotNetCliToolDir>
@@ -83,12 +83,12 @@
   </PropertyGroup>
 
   <PropertyGroup>
+    <ArcadeOverridesDir>$(RepositoryEngineeringDir)arcade-overrides/</ArcadeOverridesDir>
     <BaseOutputPath>$(ArtifactsDir)</BaseOutputPath>
     <ClonedSubmoduleGitRootDirectory Condition="'$(ClonedSubmoduleGitRootDirectory)' == ''">$(ProjectDir).git/modules/src/</ClonedSubmoduleGitRootDirectory>
-    <ToolsDir>$(ProjectDir)Tools/</ToolsDir>
-    <ToolPackageExtractDir>$(ToolsDir)source-built/</ToolPackageExtractDir>
-    <ToolsLocalDir>$(ProjectDir)tools-local/</ToolsLocalDir>
-    <TaskDirectory>$(ToolsLocalDir)tasks/</TaskDirectory>
+    <SourceBuiltSdksDir>$(ArtifactsDir)source-built-sdks/</SourceBuiltSdksDir>
+    <ToolsDir>$(RepositoryEngineeringDir)/tools/</ToolsDir>
+    <TaskDirectory>$(ToolsDir)tasks/</TaskDirectory>
 
     <XPlatTasksBinDir>$(TaskDirectory)Microsoft.DotNet.SourceBuild.Tasks.XPlat/bin/$(Configuration)/</XPlatTasksBinDir>
     <XPlatSourceBuildTasksAssembly>$(XPlatTasksBinDir)Microsoft.DotNet.SourceBuild.Tasks.XPlat.dll</XPlatSourceBuildTasksAssembly>
@@ -129,6 +129,7 @@
     <OfflineGitInfoPropsFile>$(GitInfoOfflineDir)$(RepositoryName).props</OfflineGitInfoPropsFile>
     <OfflineAllRepoPropsFile>$(GitInfoOfflineDir)AllRepoVersions.props</OfflineAllRepoPropsFile>
     <PackageReportDir>$(BaseOutputPath)prebuilt-report/</PackageReportDir>
+    <RepoProjectsDir>$(ProjectDir)/repo-projects/</RepoProjectsDir>
     <ResultingPrebuiltPackagesDir>$(PackageReportDir)prebuilt-packages/</ResultingPrebuiltPackagesDir>
     <PackageReportDataFile>$(PackageReportDir)prebuilt-usage.xml</PackageReportDataFile>
     <PoisonUsageReportFile>$(PackageReportDir)poison-usage.xml</PoisonUsageReportFile>
@@ -148,15 +149,12 @@
     <SourceBuiltArtifactsTarballUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/</SourceBuiltArtifactsTarballUrl>
     <ArchiveArtifactsTextFileName>archiveArtifacts.txt</ArchiveArtifactsTextFileName>
     <ArchiveArtifactsTextFile>$(ExternalTarballsDir)$(ArchiveArtifactsTextFileName)</ArchiveArtifactsTextFile>
-    <BaselineDataFile>$(ToolsLocalDir)prebuilt-baseline.xml</BaselineDataFile>
+    <BaselineDataFile>$(ToolsDir)prebuilt-baseline.xml</BaselineDataFile>
     <!--Exclude tests that are failing for test enabled projects like - corefx -->
     <TestExclusionsDir>$(ProjectDir)test/exclusions/</TestExclusionsDir>
     <SmokeTestsDir>$(ProjectDir)test/Microsoft.DotNet.SourceBuild.SmokeTests/</SmokeTestsDir>
   </PropertyGroup>
 
-  <!-- Import Build tools common props file where repo-independent properties are found -->
-  <Import Project="$(ToolsDir)Build.Common.props" Condition="Exists('$(ToolsDir)Build.Common.props')" />
-
   <PropertyGroup>
     <ArmEnvironmentVariables Condition="'$(ArmEnvironmentVariables)' == ''">ROOTFS_DIR=$(BaseIntermediatePath)crossrootfs/arm</ArmEnvironmentVariables>
     <ArmEnvironmentVariables Condition="'$(Platform)' == 'armel'">ROOTFS_DIR=$(BaseIntermediatePath)crossrootfs/armel</ArmEnvironmentVariables>

src/SourceBuild/tarball/content/build.proj

@@ -11,14 +11,14 @@
   <Target Name="Build" DependsOnTargets="PrepareOutput;InitBuild">
     <Message Text="Build Environment: $(Platform) $(Configuration) $(TargetOS) $(TargetRid)" />
 
-    <MSBuild Projects="repos\$(RootRepo).proj" Targets="Build" BuildInParallel="$(BuildInParallel)" StopOnFirstFailure="true" />
+    <MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj" Targets="Build" BuildInParallel="$(BuildInParallel)" StopOnFirstFailure="true" />
   </Target>
 
   <Target Name="RunRepoTests" DependsOnTargets="PrepareOutput;InitBuild">
     <Message Text="Build Environment: $(Platform) $(Configuration) $(TargetOS) $(TargetRid)" />
 
-    <MSBuild Projects="repos\$(RootRepoTests).proj" Targets="Build" Properties="PrepForTests=true;SkipEnsurePackagesCreated=true" BuildInParallel="$(BuildInParallel)" StopOnFirstFailure="true" />
-    <MSBuild Projects="repos\$(RootRepoTests).proj" Targets="Build" Properties="RunTests=true;SkipEnsurePackagesCreated=true" BuildInParallel="$(BuildInParallel)" StopOnFirstFailure="true" />
+    <MSBuild Projects="$(RepoProjectsDir)$(RootRepoTests).proj" Targets="Build" Properties="PrepForTests=true;SkipEnsurePackagesCreated=true" BuildInParallel="$(BuildInParallel)" StopOnFirstFailure="true" />
+    <MSBuild Projects="$(RepoProjectsDir)$(RootRepoTests).proj" Targets="Build" Properties="RunTests=true;SkipEnsurePackagesCreated=true" BuildInParallel="$(BuildInParallel)" StopOnFirstFailure="true" />
   </Target>
 
   <Target Name="PrepareOutput">
@@ -36,7 +36,7 @@
   </Target>
 
   <Target Name="InitBuild">
-    <MSBuild Projects="tools-local/init-build.proj" />
+    <MSBuild Projects="$(ToolsDir)init-build.proj" />
   </Target>
 
   <Target Name="Clean">
@@ -47,7 +47,7 @@
   <Target Name="ReportPrebuiltUsage"
           AfterTargets="Build"
           Condition="'$(SkipReportPrebuiltUsage)' != 'true'">
-    <MSBuild Projects="repos\$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
+    <MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj" Targets="WritePrebuiltUsageData;ReportPrebuiltUsage" />
   </Target>
 
   <!--
@@ -57,7 +57,7 @@
     XMLs based on the info put in artifacts/.
   -->
   <Target Name="RewritePrebuiltUsageReport">
-    <MSBuild Projects="repos\$(RootRepo).proj" Targets="ReportPrebuiltUsage" />
+    <MSBuild Projects="$(RepoProjectsDir)$(RootRepo).proj" Targets="ReportPrebuiltUsage" />
   </Target>
 
   <Target Name="ReportPoisonUsage"

src/SourceBuild/tarball/content/build.sh

@@ -183,7 +183,7 @@ LogDateStamp=$(date +"%m%d%H%M%S")
 if [ "$alternateTarget" == "true" ]; then
   "$CLI_ROOT/dotnet" msbuild "$SCRIPT_ROOT/build.proj" -bl:$SCRIPT_ROOT/artifacts/log/Debug/BuildTests_$LogDateStamp.binlog -flp:LogFile=$SCRIPT_ROOT/artifacts/logs/BuildTests_$LogDateStamp.log -clp:v=m ${MSBUILD_ARGUMENTS[@]} "$@"
 else
-  "$CLI_ROOT/dotnet" msbuild "$SCRIPT_ROOT/tools-local/init-build.proj" -bl:$SCRIPT_ROOT/artifacts/log/Debug/BuildXPlatTasks_$LogDateStamp.binlog -flp:LogFile=$SCRIPT_ROOT/artifacts/logs/BuildXPlatTasks_$LogDateStamp.log -t:PrepareOfflineLocalTools ${MSBUILD_ARGUMENTS[@]} "$@"
+  "$CLI_ROOT/dotnet" msbuild "$SCRIPT_ROOT/eng/tools/init-build.proj" -bl:$SCRIPT_ROOT/artifacts/log/Debug/BuildXPlatTasks_$LogDateStamp.binlog -flp:LogFile=$SCRIPT_ROOT/artifacts/logs/BuildXPlatTasks_$LogDateStamp.log -t:PrepareOfflineLocalTools ${MSBUILD_ARGUMENTS[@]} "$@"
   # kill off the MSBuild server so that on future invocations we pick up our custom SDK Resolver
   "$CLI_ROOT/dotnet" build-server shutdown
 

src/SourceBuild/tarball/content/eng/tools/generate-graphviz/generate-graphviz.proj

@@ -4,10 +4,10 @@
   <Target Name="Build">
     <ItemGroup>
       <AllRepoProjects
-        Include="$(ProjectDir)repos\*.proj"
+        Include="$(ProjectDir)repo-projects\*.proj"
         Exclude="
-          $(ProjectDir)repos\known-good.proj;
-          $(ProjectDir)repos\known-good-tests.proj" />
+          $(ProjectDir)repo-projects\known-good.proj;
+          $(ProjectDir)repo-projects\known-good-tests.proj" />
     </ItemGroup>
 
     <MSBuild

src/SourceBuild/tarball/content/eng/tools/init-build.proj

@@ -173,7 +173,7 @@
     <!-- Allow overriding of Arcade targets for SourceBuild to enable quicker
          dev turnaround for Preview 6 -->
     <ItemGroup>
-      <OverrideArcadeFiles Include="$(ProjectDir)ArcadeOverrides/**/*" />
+      <OverrideArcadeFiles Include="$(ArcadeOverridesDir)**/*" />
     </ItemGroup>
 
     <Copy

src/SourceBuild/tarball/content/repo-projects/Directory.Build.props

@@ -61,7 +61,7 @@
     <!-- _InitializeDotNetCli is used by websdk and templating to decide whether to init the SDK -->
     <EnvironmentVariables Include="_InitializeDotNetCli=$(DotNetCliToolDir)" />
     <EnvironmentVariables Include="_DotNetInstallDir=$(DotNetCliToolDir)" />
-    <EnvironmentVariables Include="_InitializeToolset=$(ProjectDir)Tools/source-built/Microsoft.DotNet.Arcade.Sdk/tools/Build.proj" Condition="'$(UseBootstrapArcade)' != 'true'" />
+    <EnvironmentVariables Include="_InitializeToolset=$(SourceBuiltSdksDir)Microsoft.DotNet.Arcade.Sdk/tools/Build.proj" Condition="'$(UseBootstrapArcade)' != 'true'" />
     <EnvironmentVariables Include="_OverrideArcadeInitializeBuildToolFramework=net7.0" />
 
     <EnvironmentVariables Include="DotNetUseShippingVersions=true" />
@@ -156,6 +156,6 @@
     <ArcadeSharedFrameworkSdkOverride Include="Microsoft.DotNet.SharedFramework.Sdk" Group="ARCADE_SHARED_FX_SDK" Version="$(arcadeOutputPackageVersion)"/>
     <ILSdkOverride Include="Microsoft.NET.Sdk.IL" Group="IL" />
     <MsBuildTraversalSdkOverride Include="Microsoft.Build.Traversal" Group="MSBUILD_TRAVERSAL" Version="2.0.2"/>
-    <WindowsDesktopSdkOverride Include="Microsoft.Net.Sdk.WindowsDesktop" Group="WINDOWS_DESKTOP" Location="$(ToolsLocalDir)EmptySdk"/>
+    <WindowsDesktopSdkOverride Include="Microsoft.Net.Sdk.WindowsDesktop" Group="WINDOWS_DESKTOP" Location="$(ToolsDir)EmptySdk"/>
   </ItemGroup>
 </Project>

src/SourceBuild/tarball/content/repo-projects/Directory.Build.targets

@@ -503,7 +503,7 @@
 
     <ItemGroup>
         <!-- Excluding Arcade here will keep it in the cache, because that's where we're running from.
-             Subsequent projects will get Arcade from Tools/source-built. -->
+             Subsequent projects will get Arcade from eng/source-built-sdks. -->
         <PackagePaths Include="@(_BuiltPackages)" Exclude="$(PackagesOutput)/Microsoft.DotNet.Arcade.Sdk.*.nupkg" />
     </ItemGroup>
 
@@ -593,28 +593,28 @@
     </ItemGroup>
 
     <ZipFileExtractToDirectory SourceArchive="%(_ToolPackage.Identity)"
-                               DestinationDirectory="$(ToolPackageExtractDir)%(_ToolPackage.Id)\"
+                               DestinationDirectory="$(SourceBuiltSdksDir)%(_ToolPackage.Id)\"
                                OverwriteDestination="true" />
     <ItemGroup>
-      <ExtractedToolFiles Include="$(ToolPackageExtractDir)%(_ToolPackage.Id)/**/*netcore*/*.dll" />
+      <ExtractedToolFiles Include="$(SourceBuiltSdksDir)%(_ToolPackage.Id)/**/*netcore*/*.dll" />
     </ItemGroup>
 
-    <Copy SourceFiles="@(ExtractedToolFiles)" DestinationFolder="$(ToolPackageExtractDir)/" />
+    <Copy SourceFiles="@(ExtractedToolFiles)" DestinationFolder="$(SourceBuiltSdksDir)/" />
 
     <!-- TODO: When unpacking using ZipFileExtractToDirectory, this executable file has the wrong
          permissions.  See https://github.com/dotnet/source-build/issues/2259 -->
     <Exec Command="chmod 755 git-clone-to-dir.sh"
       Condition=" '%(_ToolPackage.Id)' == 'Microsoft.DotNet.Arcade.Sdk' "
-      WorkingDirectory="$(ToolPackageExtractDir)%(_ToolPackage.Id)/tools/SourceBuild/" />
+      WorkingDirectory="$(SourceBuiltSdksDir)%(_ToolPackage.Id)/tools/SourceBuild/" />
 
     <!-- Allow overriding of Arcade targets for SourceBuild to enable quicker
          dev turnaround for Preview 6 -->
     <PropertyGroup>
       <ArcadeSDKToolPackagePath></ArcadeSDKToolPackagePath>
-      <ArcadeSDKToolPackagePath Condition=" '%(_ToolPackage.Id)' == 'Microsoft.DotNet.Arcade.Sdk' ">$(ToolPackageExtractDir)%(_ToolPackage.Id)/</ArcadeSDKToolPackagePath>
+      <ArcadeSDKToolPackagePath Condition=" '%(_ToolPackage.Id)' == 'Microsoft.DotNet.Arcade.Sdk' ">$(SourceBuiltSdksDir)%(_ToolPackage.Id)/</ArcadeSDKToolPackagePath>
     </PropertyGroup>
     <ItemGroup>
-      <OverrideArcadeFiles Include="$(ProjectDir)ArcadeOverrides/**/*" />
+      <OverrideArcadeFiles Include="$(ArcadeOverridesDir)**/*" />
     </ItemGroup>
 
     <Copy
@@ -684,7 +684,7 @@
       <EnvironmentVariables Include="SOURCE_BUILT_SDK_ID_%(UseSourceBuiltSdkOverride.Group)=%(UseSourceBuiltSdkOverride.Identity)" />
       <EnvironmentVariables Include="SOURCE_BUILT_SDK_VERSION_%(UseSourceBuiltSdkOverride.Group)=%(UseSourceBuiltSdkOverride.Version)" />
       <EnvironmentVariables Condition="'%(UseSourceBuiltSdkOverride.Location)' != ''" Include="SOURCE_BUILT_SDK_DIR_%(UseSourceBuiltSdkOverride.Group)=%(UseSourceBuiltSdkOverride.Location)/" />
-      <EnvironmentVariables Condition="'%(UseSourceBuiltSdkOverride.Location)' == ''" Include="SOURCE_BUILT_SDK_DIR_%(UseSourceBuiltSdkOverride.Group)=$(ToolPackageExtractDir)%(UseSourceBuiltSdkOverride.Identity)/" />
+      <EnvironmentVariables Condition="'%(UseSourceBuiltSdkOverride.Location)' == ''" Include="SOURCE_BUILT_SDK_DIR_%(UseSourceBuiltSdkOverride.Group)=$(SourceBuiltSdksDir)%(UseSourceBuiltSdkOverride.Identity)/" />
     </ItemGroup>
   </Target>
 
@@ -714,7 +714,7 @@
       <PlatformsRuntimeJsonFiles Include="$(PackagesDir)microsoft.netcore.platforms/*/runtime.json" />
 
       <!-- Add some other potential top-level project directories for a more specific report. -->
-      <ProjectDirectories Include="$(ToolsDir);$(TaskDirectory);$(BaseIntermediatePath)" />
+      <ProjectDirectories Include="$(SourceBuiltSdksDir);$(TaskDirectory);$(BaseIntermediatePath)" />
       <!-- Finally, scan entire source-build, in case project.assets.json ends up in an unexpected place. -->
       <ProjectDirectories Include="$(ProjectDir)" />
     </ItemGroup>
@@ -761,7 +761,7 @@
 
   <Target Name="GetAllProjectDirectories">
     <ItemGroup>
-      <AllRepoProjects Include="$(ProjectDir)repos\*.proj" />
+      <AllRepoProjects Include="$(RepoProjectsDir)*.proj" />
     </ItemGroup>
 
     <Message Importance="High" Text="Finding project directories..." />

src/SourceBuild/tarball/content/repo-projects/fsharp.proj

@@ -33,7 +33,7 @@
   <Target Name="SetArcadeSdkDir"
           BeforeTargets="Build">
     <ItemGroup>
-      <EnvironmentVariables Include="ArcadeSdkDir=$(ToolPackageExtractDir)%(UseSourceBuiltSdkOverride.Identity)/" />
+      <EnvironmentVariables Include="ArcadeSdkDir=$(SourceBuiltSdksDir)%(UseSourceBuiltSdkOverride.Identity)/" />
     </ItemGroup>
   </Target>
 

src/SourceBuild/tarball/content/repo-projects/msbuild.proj

@@ -57,7 +57,7 @@
     </ItemGroup>
 
     <PropertyGroup>
-      <CentralVersionsSdkDir>$(ToolPackageExtractDir)%(_CentralVersionsToolPackage.Id)/</CentralVersionsSdkDir>
+      <CentralVersionsSdkDir>$(SourceBuiltSdksDir)%(_CentralVersionsToolPackage.Id)/</CentralVersionsSdkDir>
     </PropertyGroup>
 
     <Message Importance="High" Text="Setting up SDK package for UseSourceBuiltSdkOverride: %(_CentralVersionsToolPackage.Filename)" />