diff --git a/src/SourceBuild/tarball/content/repos/source-build-reference-packages.proj b/src/SourceBuild/tarball/content/repos/source-build-reference-packages.proj
index 69ab564a2..a4fb27340 100644
--- a/src/SourceBuild/tarball/content/repos/source-build-reference-packages.proj
+++ b/src/SourceBuild/tarball/content/repos/source-build-reference-packages.proj
@@ -2,7 +2,9 @@
   <Import Project="Sdk.props" Sdk="Microsoft.NET.Sdk" />
 
   <PropertyGroup>
-    <BuildCommand>$(StandardSourceBuildCommand) $(StandardSourceBuildArgs)</BuildCommand>
+    <LocalNuGetPackageCacheDirectory>$(BaseIntermediatePath)source-build-reference-package-cache</LocalNuGetPackageCacheDirectory>
+
+    <BuildCommand>$(StandardSourceBuildCommand) $(StandardSourceBuildArgs) /p:LocalNuGetPackageCacheDirectory=$(LocalNuGetPackageCacheDirectory)</BuildCommand>
 
     <NuGetConfigFile>$(ProjectDirectory)NuGet.config</NuGetConfigFile>
     <GlobalJsonFile>$(ProjectDirectory)global.json</GlobalJsonFile>
@@ -16,5 +18,15 @@
     <UseSourceBuiltSdkOverride Include="@(ArcadeBootstrapSdkOverride)" />
   </ItemGroup>
 
+  <Target Name="AddLocalNuGetPackageCacheDirectory" BeforeTargets="Build">
+    <MakeDir Condition="'$(LocalNuGetPackageCacheDirectory)' != ''"
+             Directories="$(LocalNuGetPackageCacheDirectory)" />
+
+    <AddSourceToNuGetConfig
+      NuGetConfigFile="$(NuGetConfigFile)"
+      SourceName="source-build-reference-package-cache"
+      SourcePath="$(LocalNuGetPackageCacheDirectory)" />
+  </Target>
+
   <Import Project="Sdk.targets" Sdk="Microsoft.NET.Sdk" />
 </Project>