Move top-level packages dir into prereqs (#15140)

src/SourceBuild/tarball/content/.gitignore

@@ -1,8 +1,8 @@
 /.dotnet
 /artifacts
-/packages
 /eng/tools/**/bin
 /eng/tools/**/obj
+/prereqs/packages
 /src/linker/src/ILLink.Tasks/ILLink.Tasks.nuspec
 /src/nuget-client/NuGet.config
 /test/Microsoft.DotNet.SourceBuild.SmokeTests/bin

src/SourceBuild/tarball/content/Directory.Build.props

@@ -39,8 +39,9 @@
     <DotNetCliToolDir Condition="'$(DotNetCliToolDir)' == '' and '$(DOTNET_INSTALL_DIR)' != ''">$([MSBuild]::NormalizeDirectory('$(DOTNET_INSTALL_DIR)'))</DotNetCliToolDir>
     <DotNetCliToolDir Condition="'$(DotNetCliToolDir)' == ''">$(ProjectDir).dotnet/</DotNetCliToolDir>
     <DotnetToolCommand>$(DotNetCliToolDir)dotnet</DotnetToolCommand>
+    <PrereqsPackagesDir>$(ProjectDir)prereqs/packages/</PrereqsPackagesDir>
     <PackagesDir Condition="'$(NuGetPackageRoot)' != ''">$(NuGetPackageRoot)</PackagesDir>
-    <PackagesDir Condition="'$(PackagesDir)' == ''">$(ProjectDir)packages/restored/</PackagesDir>
+    <PackagesDir Condition="'$(PackagesDir)' == ''">$(PrereqsPackagesDir)restored/</PackagesDir>
     <ArcadeBootstrapPackageDir>$(PackagesDir)ArcadeBootstrapPackage/</ArcadeBootstrapPackageDir>
     <!-- if we're not currently building,  Visual Studio will still set this -->
     <SDK_VERSION Condition="'$(SDK_VERSION)' == ''">$(NETCoreSdkVersion)</SDK_VERSION>
@@ -105,9 +106,9 @@
     <SourceBuiltBlobFeedDir>$(IntermediatePath)blob-feed/</SourceBuiltBlobFeedDir>
     <SourceBuiltPackagesPath>$(SourceBuiltBlobFeedDir)packages/</SourceBuiltPackagesPath>
     <SourceBuiltAssetsDir>$(SourceBuiltBlobFeedDir)assets/</SourceBuiltAssetsDir>
-    <PrebuiltPackagesPath>$(ProjectDir)packages/prebuilt/</PrebuiltPackagesPath>
+    <PrebuiltPackagesPath>$(PrereqsPackagesDir)prebuilt/</PrebuiltPackagesPath>
-    <PreviouslyRestoredPackagesPath>$(ProjectDir)packages/previouslyRestored/</PreviouslyRestoredPackagesPath>
+    <PreviouslyRestoredPackagesPath>$(PrereqsPackagesDir)previouslyRestored/</PreviouslyRestoredPackagesPath>
-    <PrebuiltSourceBuiltPackagesPath>$(ProjectDir)packages/previously-source-built/</PrebuiltSourceBuiltPackagesPath>
+    <PrebuiltSourceBuiltPackagesPath>$(PrereqsPackagesDir)previously-source-built/</PrebuiltSourceBuiltPackagesPath>
     <PrebuiltSourceBuiltPackagesPath Condition="'$(CustomPrebuiltSourceBuiltPackagesPath)' != ''">$(CustomPrebuiltSourceBuiltPackagesPath)/</PrebuiltSourceBuiltPackagesPath>
     <SourceBuiltTarBallPath>$(OutputPath)</SourceBuiltTarBallPath>
     <SourceBuiltToolsetDir>$(LocalBlobStorageRoot)Sdk/</SourceBuiltToolsetDir>
@@ -138,8 +139,8 @@
     <PoisonedReportFile>$(PackageReportDir)poisoned.txt</PoisonedReportFile>
     <ConflictingPackageReportDir>$(BaseOutputPath)conflict-report/</ConflictingPackageReportDir>
     <PrebuiltBurndownDataFile>$(PackageReportDir)PrebuiltBurndownData.csv</PrebuiltBurndownDataFile>
-    <ExternalTarballsDir>$(ProjectDir)packages/archive/</ExternalTarballsDir>
+    <ExternalTarballsDir>$(PrereqsPackagesDir)archive/</ExternalTarballsDir>
-    <ReferencePackagesDir>$(ProjectDir)packages/reference/</ReferencePackagesDir>
+    <ReferencePackagesDir>$(PrereqsPackagesDir)reference/</ReferencePackagesDir>
     <SourceBuiltArtifactsTarballName>Private.SourceBuilt.Artifacts</SourceBuiltArtifactsTarballName>
     <SourceBuiltPrebuiltsTarballName>Private.SourceBuilt.Prebuilts</SourceBuiltPrebuiltsTarballName>
     <SourceBuiltArtifactsTarballUrl>https://dotnetcli.azureedge.net/source-built-artifacts/assets/</SourceBuiltArtifactsTarballUrl>

src/SourceBuild/tarball/content/build.sh

@@ -23,6 +23,10 @@ CUSTOM_REF_PACKAGES_DIR=''
 CUSTOM_PACKAGES_DIR=''
 alternateTarget=false
 runningSmokeTests=false
+packagesDir="$SCRIPT_ROOT/prereqs/packages/"
+packagesArchiveDir="${packagesDir}archive/"
+packagesRestoredDir="${packagesDir}restored/"
+packagesPreviouslySourceBuiltDir="${packagesDir}previously-source-built/"
 CUSTOM_SDK_DIR=''
 
 while :; do
@@ -95,14 +99,14 @@ if [ "$CUSTOM_PACKAGES_DIR" != "" ]; then
   fi
 fi
 
-if [ -f "$SCRIPT_ROOT/packages/archive/archiveArtifacts.txt" ]; then
+if [ -f "${packagesArchiveDir}archiveArtifacts.txt" ]; then
   ARCHIVE_ERROR=0
   if [ ! -d "$SCRIPT_ROOT/.dotnet" ] && [ "$CUSTOM_SDK_DIR" == "" ]; then
     echo "ERROR: SDK not found at $SCRIPT_ROOT/.dotnet"
     ARCHIVE_ERROR=1
   fi
-  if [ ! -f $SCRIPT_ROOT/packages/archive/Private.SourceBuilt.Artifacts*.tar.gz ] && [ "$CUSTOM_PACKAGES_DIR" == "" ]; then
+  if [ ! -f ${packagesArchiveDir}Private.SourceBuilt.Artifacts*.tar.gz ] && [ "$CUSTOM_PACKAGES_DIR" == "" ]; then
-    echo "ERROR: Private.SourceBuilt.Artifacts artifact not found at $SCRIPT_ROOT/packages/archive/ - Either run prep.sh or pass --with-packages parameter"
+    echo "ERROR: Private.SourceBuilt.Artifacts artifact not found at $packagesArchiveDir - Either run prep.sh or pass --with-packages parameter"
     ARCHIVE_ERROR=1
   fi
   if [ $ARCHIVE_ERROR == 1 ]; then
@@ -131,14 +135,13 @@ else
 fi
 
 packageVersionsPath=''
-restoredPackagesDir="$SCRIPT_ROOT/packages/restored"
 
 if [[ "$CUSTOM_PACKAGES_DIR" != "" && -f "$CUSTOM_PACKAGES_DIR/PackageVersions.props" ]]; then
   packageVersionsPath="$CUSTOM_PACKAGES_DIR/PackageVersions.props"
-elif [ -d "$SCRIPT_ROOT/packages/archive" ]; then
+elif [ -d "$packagesArchiveDir" ]; then
-  sourceBuiltArchive=`find $SCRIPT_ROOT/packages/archive -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'`
+  sourceBuiltArchive=`find $packagesArchiveDir -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'`
-  if [ -f "$SCRIPT_ROOT/packages/previously-source-built/PackageVersions.props" ]; then
+  if [ -f "${packagesPreviouslySourceBuiltDir}}PackageVersions.props" ]; then
-    packageVersionsPath=$SCRIPT_ROOT/packages/previously-source-built/PackageVersions.props
+    packageVersionsPath=${packagesPreviouslySourceBuiltDir}PackageVersions.props
   elif [ -f "$sourceBuiltArchive" ]; then
     tar -xzf "$sourceBuiltArchive" -C /tmp PackageVersions.props
     packageVersionsPath=/tmp/PackageVersions.props
@@ -147,7 +150,7 @@ fi
 
 if [ ! -f "$packageVersionsPath" ]; then
   echo "Cannot find PackagesVersions.props.  Debugging info:"
-  echo "  Attempted archive path: $SCRIPT_ROOT/packages/archive"
+  echo "  Attempted archive path: $packagesArchiveDir"
   echo "  Attempted custom PVP path: $CUSTOM_PACKAGES_DIR/PackageVersions.props"
   exit 1
 fi
@@ -162,7 +165,7 @@ if [[ $arcadeSdkLine =~ $versionPattern ]]; then
   # projects overwrite this so that they use the source-built Arcade SDK instad.
   export SOURCE_BUILT_SDK_ID_ARCADE=Microsoft.DotNet.Arcade.Sdk
   export SOURCE_BUILT_SDK_VERSION_ARCADE=$ARCADE_BOOTSTRAP_VERSION
-  export SOURCE_BUILT_SDK_DIR_ARCADE=$restoredPackagesDir/ArcadeBootstrapPackage/microsoft.dotnet.arcade.sdk/$ARCADE_BOOTSTRAP_VERSION
+  export SOURCE_BUILT_SDK_DIR_ARCADE=$packagesRestoredDir/ArcadeBootstrapPackage/microsoft.dotnet.arcade.sdk/$ARCADE_BOOTSTRAP_VERSION
 fi
 
 sourceLinkLine=`grep -m 1 'MicrosoftSourceLinkCommonVersion' "$packageVersionsPath"`
@@ -174,7 +177,7 @@ fi
 echo "Found bootstrap SDK $SDK_VERSION, bootstrap Arcade $ARCADE_BOOTSTRAP_VERSION, bootstrap SourceLink $SOURCE_LINK_BOOTSTRAP_VERSION"
 
 export DOTNET_CLI_TELEMETRY_OPTOUT=1
-export NUGET_PACKAGES=$restoredPackagesDir/
+export NUGET_PACKAGES=$packagesRestoredDir/
 
 LogDateStamp=$(date +"%m%d%H%M%S")
 

src/SourceBuild/tarball/content/prep.sh

@@ -52,14 +52,15 @@ fi
 
 # Check if Private.SourceBuilt artifacts archive exists
 artifactsBaseFileName="Private.SourceBuilt.Artifacts"
-if [ -f $SCRIPT_ROOT/packages/archive/$artifactsBaseFileName.*.tar.gz ]; then
+packagesArchiveDir="$SCRIPT_ROOT/prereqs/packages/archive/"
+if [ -f ${packagesArchiveDir}${artifactsBaseFileName}.*.tar.gz ]; then
     echo "  Private.SourceBuilt.Artifacts.*.tar.gz exists...it will not be downloaded"
     downloadArtifacts=false
 fi
 
 # Check if Private.SourceBuilt prebuilts archive exists
 prebuiltsBaseFileName="Private.SourceBuilt.Prebuilts"
-if [ -f $SCRIPT_ROOT/packages/archive/$prebuiltsBaseFileName.*.tar.gz ]; then
+if [ -f ${packagesArchiveDir}${prebuiltsBaseFileName}.*.tar.gz ]; then
     echo "  Private.SourceBuilt.Prebuilts.*.tar.gz exists...it will not be downloaded"
     downloadPrebuilts=false
 fi
@@ -85,7 +86,7 @@ function DownloadArchive {
     if [[ $archiveVersionLine =~ $versionPattern ]]; then
         archiveUrl="${sourceBuiltArtifactsTarballUrl}${baseFileName}.${BASH_REMATCH[1]}.tar.gz"
         echo "  Downloading source-built $archiveType from $archiveUrl..."
-        (cd $SCRIPT_ROOT/packages/archive/ && curl --retry 5 -O $archiveUrl)
+        (cd $packagesArchiveDir && curl --retry 5 -O $archiveUrl)
     elif [ "$isRequired" == "true" ]; then
       echo "  ERROR: $notFoundMessage"
       exit -1
@@ -128,13 +129,13 @@ if [ "$buildBootstrap" == "true" ]; then
 
     # Get PackageVersions.props from existing prev-sb archive
     echo "  Retrieving PackageVersions.props from existing archive"
-    sourceBuiltArchive=`find $SCRIPT_ROOT/packages/archive -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'`
+    sourceBuiltArchive=`find $packagesArchiveDir -maxdepth 1 -name 'Private.SourceBuilt.Artifacts*.tar.gz'`
     if [ -f "$sourceBuiltArchive" ]; then
         tar -xzf "$sourceBuiltArchive" -C $workingDir PackageVersions.props
     fi
 
     # Run restore on project to initiate download of bootstrap packages
-    $DOTNET_SDK_PATH/dotnet restore $workingDir/buildBootstrapPreviouslySB.csproj /bl:artifacts/prep/bootstrap.binlog /fileLoggerParameters:LogFile=artifacts/prep/bootstrap.log /p:ArchiveDir="$SCRIPT_ROOT/packages/archive/" /p:BootstrapOverrideVersionsProps="$SCRIPT_ROOT/eng/bootstrap/OverrideBootstrapVersions.props"
+    $DOTNET_SDK_PATH/dotnet restore $workingDir/buildBootstrapPreviouslySB.csproj /bl:artifacts/prep/bootstrap.binlog /fileLoggerParameters:LogFile=artifacts/prep/bootstrap.log /p:ArchiveDir="$packagesArchiveDir" /p:BootstrapOverrideVersionsProps="$SCRIPT_ROOT/eng/bootstrap/OverrideBootstrapVersions.props"
 
     # Remove working directory
     rm -rf $workingDir

src/SourceBuild/tarball/content/test/Microsoft.DotNet.SourceBuild.SmokeTests/README.md

@@ -7,4 +7,4 @@
 
 Some prerelease scenarios, usually security updates, require non-source-built packages which are not publicly available.
 Specify the directory where these packages can be found via the `SMOKE_TESTS_PREREQS_PATH` environment variable when running tests via `build.sh --run-smoke-test` e.g.
-`SMOKE_TESTS_PREREQS_PATH=packages/smoke-test-prereqs`.
+`SMOKE_TESTS_PREREQS_PATH=prereqs/packages/smoke-test-prereqs`.