Download blobs from MSRC storage & Add CredProvider to Dockerfiles (#4753)

* Download core-setup files from MSRC storage

Add support for downloading core-setup files from an authenticated endpoint

* Update GenerateLayout.targets

* Adding changes to pass SAS token around

* Add credential provider to Dockerfile's

* Update after PR feedback

* Add comments about approach

* Copy NuGet config variables

eng/build.yml

@@ -40,6 +40,11 @@ phases:
       - _TeamName: Roslyn-Project-System
       - _BuildArgs: ''
       - _SignType: test
+      - _DOTNETCLIMSRC_READ_SAS_TOKEN: ''
+
+      - ${{ if and(eq(variables['System.TeamProject'], 'internal'), contains(variables['Build.SourceBranch'], 'internal')) }}:
+        - group: DotNet-MSRC-Storage
+        - _DOTNETCLIMSRC_READ_SAS_TOKEN: $(dotnetclimsrc-read-sas-token)
 
       - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
         - group: DotNet-Symbol-Server-PATs
@@ -79,6 +84,7 @@ phases:
         env:
           DOTNET_CLI_UI_LANGUAGE: $(_DOTNET_CLI_UI_LANGUAGE)
           DropSuffix: $(_DropSuffix)
+          DOTNETCLIMSRC_READ_SAS_TOKEN: $(_DOTNETCLIMSRC_READ_SAS_TOKEN)
 
     - ${{ if ne(parameters.agentOs, 'Windows_NT') }}:
       - script: 'eng/setbuildinfo.sh $(_BuildArchitecture) $(_BuildConfig)'
@@ -98,6 +104,7 @@ phases:
         displayName: Build
         env:
           DropSuffix: $(_DropSuffix)
+          DOTNETCLIMSRC_READ_SAS_TOKEN: $(_DOTNETCLIMSRC_READ_SAS_TOKEN)
 
     - ${{ if or(eq(parameters.agentOs, 'Darwin'), eq(parameters.agentOs, 'FreeBSD')) }}:
       - script: ./build.sh
@@ -108,6 +115,8 @@ phases:
                   $(_BuildArgs)
                   $(_AdditionalBuildParameters)
         displayName: Build
+        env:
+          DOTNETCLIMSRC_READ_SAS_TOKEN: $(_DOTNETCLIMSRC_READ_SAS_TOKEN)
 
     - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), eq(parameters.agentOs, 'Windows_NT')) }}:
       - task: NuGetCommand@2

eng/docker/alpine.3.6/Dockerfile

@@ -18,10 +18,17 @@ RUN echo 'code_executor ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers
 RUN chmod -R a+rwx /usr/local
 RUN chmod -R a+rwx /home
 
-# Set user to the one we just created
-USER ${USER_ID}
-
 # Set working directory 
 ARG WORK_DIR
 WORKDIR ${WORK_DIR}
 
+# Set up Azure Artifacts credential provider
+# We download the installer and execute it using ${USER_ID} so that the installer
+# put the NuGet plugins in the correct $HOME/.nuget folder.
+RUN curl -O https://raw.githubusercontent.com/Microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh
+RUN chmod +x ./installcredprovider.sh
+
+# Set user to the one we just created
+USER ${USER_ID}
+
+RUN ./installcredprovider.sh

eng/docker/rhel.6/Dockerfile

@@ -30,3 +30,5 @@ ENV LD_LIBRARY_PATH /usr/local/lib
 ARG WORK_DIR
 WORKDIR ${WORK_DIR}
 
+# Set up Azure Artifacts credential provider
+RUN wget -qO- https://raw.githubusercontent.com/Microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh | bash

eng/docker/rhel/Dockerfile

@@ -22,3 +22,6 @@ USER ${USER_ID}
 # Set working directory
 ARG WORK_DIR
 WORKDIR ${WORK_DIR}
+
+# Set up Azure Artifacts credential provider
+RUN wget -qO- https://raw.githubusercontent.com/Microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh | bash

eng/docker/ubuntu.16.04/Dockerfile

@@ -49,9 +49,17 @@ RUN chmod -R a+rwx /usr/local
 RUN chmod -R a+rwx /home
 RUN chmod -R 755 /usr/lib/sudo
 
-# Set user to the one we just created
-USER ${USER_ID}
-
 # Set working directory 
 ARG WORK_DIR
 WORKDIR ${WORK_DIR}
+
+# Set up Azure Artifacts credential provider. 
+# We download the installer and execute it using ${USER_ID} so that the installer
+# put the NuGet plugins in the correct $HOME/.nuget folder.
+RUN curl -O https://raw.githubusercontent.com/Microsoft/artifacts-credprovider/master/helpers/installcredprovider.sh 
+RUN chmod +x ./installcredprovider.sh
+
+# Set user to the one we just created
+USER ${USER_ID}
+
+RUN ./installcredprovider.sh

eng/dockerrun.sh

@@ -142,7 +142,13 @@ docker run $INTERACTIVE -t --rm --sig-proxy=true \
     -e BUILD_SOURCEBRANCH \
     -e BUILD_BUILDNUMBER \
     -e BUILD_SOURCEVERSION \
+    -e SYSTEM_TEAMPROJECT \
+    -e DOTNECLIMSRC_READ_SAS_TOKEN \
     -e AGENT_JOBNAME \
     -e AGENT_OS \
+    -e VSS_NUGET_URI_PREFIXES \
+    -e VSS_NUGET_ACCESSTOKEN \
+    -e DOTNET_SYSTEM_NET_HTTP_USESOCKETSHTTPHANDLER=0 \
+    -e NUGET_CREDENTIALPROVIDER_SESSIONTOKENCACHE_ENABLED=true \
     $DOTNET_BUILD_CONTAINER_TAG \
     $BUILD_COMMAND "$@"

src/redist/targets/GenerateLayout.targets

@@ -18,8 +18,17 @@
   <Target Name="SetupBundledComponents" DependsOnTargets="GetCurrentRuntimeInformation;SetupFileExtensions;SetSdkVersionInfo;SetBuildDefaults">
     <PropertyGroup>
       <SdkOutputDirectory>$(RedistLayoutPath)sdk\$(SdkVersion)\</SdkOutputDirectory>
+      
+      <InternalBuild Condition="$(SYSTEM_TEAMPROJECT) == 'internal' and $(BUILD_SOURCEBRANCH.ToLower().contains('internal'))">true</InternalBuild>
+      <CoreSetupBlobAccessTokenParam Condition="'$(InternalBuild)' == 'true'">$(DOTNETCLIMSRC_READ_SAS_TOKEN)</CoreSetupBlobAccessTokenParam>
+
+      <CoreSetupBlobRootUrl Condition="'$(CoreSetupBlobRootUrl)' == '' and '$(InternalBuild)' == 'true'">https://dotnetclimsrc.blob.core.windows.net/dotnet/</CoreSetupBlobRootUrl>
       <CoreSetupBlobRootUrl Condition="'$(CoreSetupBlobRootUrl)' == ''">https://dotnetcli.azureedge.net/dotnet/</CoreSetupBlobRootUrl>
+
+      <DotnetExtensionsBlobRootUrl Condition="'$(DotnetExtensionsBlobRootUrl)' == '' and '$(InternalBuild)' == 'true'">https://dotnetclimsrc.blob.core.windows.net/dotnet/</DotnetExtensionsBlobRootUrl>
       <DotnetExtensionsBlobRootUrl Condition="'$(DotnetExtensionsBlobRootUrl)' == ''">https://dotnetcli.blob.core.windows.net/dotnet/</DotnetExtensionsBlobRootUrl>
+      
+      <DotnetToolsetBlobRootUrl Condition="'$(DotnetToolsetBlobRootUrl)' == '' and '$(InternalBuild)' == 'true'">https://dotnetclimsrc.blob.core.windows.net/dotnet/</DotnetToolsetBlobRootUrl>
       <DotnetToolsetBlobRootUrl Condition="'$(DotnetToolsetBlobRootUrl)' == ''">https://dotnetfeed.blob.core.windows.net/dotnet-toolset/</DotnetToolsetBlobRootUrl>
       
       <CoreSetupRid Condition="'$(CoreSetupRid)' == ''">$(HostRid)</CoreSetupRid>
@@ -277,7 +286,7 @@
     </ItemGroup>
   
     <DownloadFile Condition=" '@(ComponentToDownload)' != '' And '%(ComponentToDownload.ShouldDownload)' == 'true'"
-                  Uri="%(ComponentToDownload.BaseUrl)/%(ComponentToDownload.DownloadFileName)"
+                  Uri="%(ComponentToDownload.BaseUrl)/%(ComponentToDownload.DownloadFileName)%(ComponentToDownload.AccessToken)"
                   DestinationPath="%(ComponentToDownload.DownloadDestination)" />
 
     <ItemGroup>