Changes for core-sdk to move to yaml stages

.vsts-ci.yml

@@ -12,10 +12,20 @@ variables:
   value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json
 - name: _ArcadePublishBlobFeedUrl
   value: https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
+- name: _DotNetArtifactsCategory
+  value: .NETCoreValidation # temporary variable name, so that we can test this change.
+- name: _DotNetValidationArtifactsCategory
+  value: .NETCoreValidation
+- name: _PublishUsingPipelines
+  value: false
 - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
   - group: DotNet-DotNetCli-Storage
   - group: DotNet-Blob-Feed
+  - name: _PublishUsingPipelines
+    value: true
 
+stages:
+- stage: build
   jobs:
   - template: /eng/build.yml
     parameters:
@@ -271,11 +281,51 @@ jobs:
   #           _AdditionalBuildParameters: '/p:DisableSourceLink=true /p:DISABLE_CROSSGEN=true'
 
   - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-  - job: Copy_SDK_To_Latest
+    - template: /eng/common/templates/job/publish-build-assets.yml
+      parameters:
         dependsOn:
           - Windows_NT
           - Linux
           - Darwin
+        publishUsingPipelines: true
+        pool:
+          vmImage: vs2017-win2016
+
+#  - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+#    - template: /eng/common/templates/job/generate-graph-files.yml
+#      parameters:
+#        continueOnError: true
+#        includeToolset: false
+#        dependsOn:
+#          - Asset_Registry_Publish
+#        pool:
+#          vmImage: vs2017-win2016
+
+- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+  - template: eng\common\templates\post-build\post-build.yml
+    parameters:
+      # Symbol validation isn't being very reliable lately. This should be enabled back
+      # once this issue is resolved: https://github.com/dotnet/arcade/issues/2871
+      enableSymbolValidation: false
+      # This repo doesn't produce any signed packages.
+      enableSigningValidation: false
+      publishInstallersAndChecksums: true
+      enableSourceLinkValidation: false # Disabling for testing in internal-only branc
+
+- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest'), not(contains(variables['Build.SourceBranch'], 'refs/heads/internal/'))) }}:
+  - stage: copy_to_latest
+    displayName: Copy to latest
+    dependsOn:
+      # This will run only after all the publishing stages have run and have been validated.
+      # These stages are introduced in the eng/common/templates/post-build/channels YAML templates
+      - NetCore_Dev31_Publish
+      - NetCore_Dev30_Publish
+      - NetCore_Dev5_Publish
+      - NetCore_Release30_Publish
+      - NetCore_Release31_Publish
+      - PVR_Publish
+    jobs:
+    - job: Copy_SDK_To_Latest
       pool:
         name: Hosted VS2017
       condition: succeeded()
@@ -283,29 +333,8 @@ jobs:
         - group: DotNet-DotNetCli-Storage
       steps:
         - script: eng/CopyToLatest.cmd
-                /p:DotnetPublishSdkAssetsBlobFeedUrl=$(_PublishBlobFeedUrl)
+                  /p:DotnetPublishSdkAssetsBlobFeedUrl=https://dotnetcli.blob.core.windows.net/riarenas/index.json
                   /p:DotNetPublishSdkAssetsBlobFeedKey=$(dotnetcli-storage-key)
-                /p:DotnetPublishChecksumsBlobFeedUrl=$(_PublishChecksumsBlobFeedUrl)
+                  /p:DotnetPublishChecksumsBlobFeedUrl=https://dotnetclichecksums.blob.core.windows.net/riarenas/index.json
                   /p:DotNetPublishChecksumsBlobFeedKey=$(dotnetclichecksums-storage-key)
           displayName: Copy to latest
-
-- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-  - template: /eng/common/templates/job/publish-build-assets.yml
-    parameters:
-      dependsOn:
-        - Windows_NT
-        - Linux
-        - Darwin
-      enablePublishBuildArtifacts: true
-      pool:
-        vmImage: vs2017-win2016
-
-- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-  - template: /eng/common/templates/job/generate-graph-files.yml
-    parameters:
-      continueOnError: true
-      includeToolset: false
-      dependsOn:
-        - Asset_Registry_Publish
-      pool:
-        vmImage: vs2017-win2016

build.sh

@@ -46,7 +46,7 @@ done
 
 dockerbuild()
 {
-    BUILD_COMMAND=/opt/code/run-build.sh $DIR/eng/dockerrun.sh --non-interactive "$@"
+    BUILD_COMMAND=$DIR/run-build.sh $DIR/eng/dockerrun.sh --non-interactive "$@"
 }
 
 # Check if we need to build in docker

eng/Publishing.props

@@ -9,9 +9,7 @@
     <BlobStoragePartialRelativePath>$(Product)</BlobStoragePartialRelativePath>
     <BlobStoragePartialRelativePath Condition="'$(IsNotOrchestratedPublish)' == 'false'">assets/$(Product)</BlobStoragePartialRelativePath>
     <ChecksumsFeedUrl>$(DotnetPublishChecksumsBlobFeedUrl)</ChecksumsFeedUrl>
-    <ChecksumsAccountKey>$(DotNetPublishChecksumsBlobFeedKey)</ChecksumsAccountKey>
     <SdkAssetsFeedUrl>$(DotnetPublishSdkAssetsBlobFeedUrl)</SdkAssetsFeedUrl>
-    <SdkAssetsAzureAccountKey>$(DotNetPublishSdkAssetsBlobFeedKey)</SdkAssetsAzureAccountKey>
 
     <PublishSdkAssetsAndChecksumsToBlob>false</PublishSdkAssetsAndChecksumsToBlob>
     <PublishSdkAssetsAndChecksumsToBlob Condition=" '$(ChecksumsFeedUrl)' != '' and '$(SdkAssetsFeedUrl)' != '' ">true</PublishSdkAssetsAndChecksumsToBlob>
@@ -33,6 +31,9 @@
       <!-- Property AssetManifestFilePath will be reassigned by the Arcade SDK, so use a different name (DotNetAssetManifestFilePath) -->
       <DotNetAssetManifestFilePath>$(ArtifactsLogDir)AssetManifest\$(AssetManifestFileName).xml</DotNetAssetManifestFilePath>
       <ChecksumsAssetManifestFilePath>$(ArtifactsLogDir)AssetManifest\$(ChecksumsAssetManifestFileName).xml</ChecksumsAssetManifestFilePath>
+
+      <DotnetTempWorkingDirectory>$(ArtifactsDir)..\DotnetAssetsTmpDir\$([System.Guid]::NewGuid())</DotnetTempWorkingDirectory>
+      <ChecksumTempWorkingDirectory>$(ArtifactsDir)..\ChecksumAssetsTmpDir\$([System.Guid]::NewGuid())</ChecksumTempWorkingDirectory>
   </PropertyGroup>
 
   <ItemGroup>
@@ -45,9 +46,7 @@
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.msi" />
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.nupkg" />
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.cab" />
-    <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.svg" Condition=" '$(PublishBinariesAndBadge)' == 'true' ">
-      <ExcludeFromManifest>true</ExcludeFromManifest>
-    </SdkAssetsToPublish>
+    <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.svg" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
     <SdkNonShippingAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.nupkg" />
     <SdkNonShippingAssetsToPublish Include="$(ArtifactsNonShippingPackagesDir)*.swr" />
     <CheckSumsToPublish Include="$(ArtifactsShippingPackagesDir)*.sha" />
@@ -74,9 +73,10 @@
       </ChecksumsToPushToBlobFeed>
     </ItemGroup>
 
-    <PushToBlobFeed
-      ExpectedFeedUrl="$(SdkAssetsFeedUrl)"
-      AccountKey="$(SdkAssetsAzureAccountKey)"
+    <MakeDir Directories="$(DotnetTempWorkingDirectory)"/>
+    <MakeDir Directories="$(ChecksumTempWorkingDirectory)"/>
+
+    <PushToAzureDevOpsArtifacts
       ItemsToPush="@(SdkAssetsToPushToBlobFeed)"
       ManifestBuildData="Location=$(SdkAssetsFeedUrl)"
       ManifestRepoUri="$(BUILD_REPOSITORY_URI)"
@@ -84,11 +84,10 @@
       ManifestBuildId="$(BUILD_BUILDNUMBER)"
       ManifestCommit="$(BUILD_SOURCEVERSION)"
       AssetManifestPath="$(DotNetAssetManifestFilePath)"
-      PublishFlatContainer="true" />
+      PublishFlatContainer="true"
+      AssetsTemporaryDirectory="$(DotnetTempWorkingDirectory)" />
 
-    <PushToBlobFeed
-      ExpectedFeedUrl="$(ChecksumsFeedUrl)"
-      AccountKey="$(ChecksumsAccountKey)"
+    <PushToAzureDevOpsArtifacts
       ItemsToPush="@(ChecksumsToPushToBlobFeed)"
       ManifestBuildData="Location=$(ChecksumsFeedUrl)"
       ManifestRepoUri="$(BUILD_REPOSITORY_URI)"
@@ -96,6 +95,23 @@
       ManifestBuildId="$(BUILD_BUILDNUMBER)"
       ManifestCommit="$(BUILD_SOURCEVERSION)"
       AssetManifestPath="$(ChecksumsAssetManifestFilePath)"
-      PublishFlatContainer="true"/>
+      PublishFlatContainer="true"
+      AssetsTemporaryDirectory="$(ChecksumTempWorkingDirectory)" />
+
+    <Copy
+      SourceFiles="$(DotNetAssetManifestFilePath)"
+      DestinationFolder="$(DotnetTempWorkingDirectory)\$(AssetManifestFileName)" />
+
+    <Copy
+      SourceFiles="$(ChecksumsAssetManifestFilePath)"
+      DestinationFolder="$(ChecksumTempWorkingDirectory)\$(ChecksumsAssetManifestFileName)" />
+
+    <Message
+      Text="##vso[artifact.upload containerfolder=AssetManifests;artifactname=AssetManifests]$(DotnetTempWorkingDirectory)/$(AssetManifestFileName)"
+      Importance="high" />
+
+    <Message
+      Text="##vso[artifact.upload containerfolder=AssetManifests;artifactname=AssetManifests]$(ChecksumTempWorkingDirectory)/$(ChecksumsAssetManifestFileName)"
+      Importance="high" />
   </Target>
 </Project>

eng/build.yml

@@ -14,13 +14,19 @@ parameters:
   # Job timeout
   timeoutInMinutes: 180
 
+  # Publish using pipelines
+  enablePublishUsingPipelines: true
+
 phases:
 - template: /eng/common/templates/job/job.yml
   parameters:
     name: ${{ parameters.agentOs }}
     ${{ if eq(parameters.agentOs, 'Windows_NT') }}:
       enableMicrobuild: true
+    enablePublishBuildArtifacts: true
     enablePublishBuildAssets: true
+    enablePublishTestResults: true
+    enablePublishUsingPipelines: ${{parameters.enablePublishUsingPipelines}}
     enableTelemetry: true
     helixRepo: dotnet/core-sdk
     pool: ${{ parameters.pool }}
@@ -38,20 +44,22 @@ phases:
       - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
         - group: DotNet-Symbol-Server-PATs
         - group: DotNet-HelixApi-Access
+        - group: DotNet-Blob-Feed
         - _DotNetPublishToBlobFeed: true
         - _PushToVSFeed: true
         - _SignType: real
         - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
                   /p:DotNetPublishToBlobFeed=$(_DotNetPublishToBlobFeed)
+                  /p:DotNetPublishBlobFeedKey=$(dotnetfeed-storage-access-key-1)
+                  /p:DotNetPublishBlobFeedUrl=$(PB_PublishBlobFeedUrl)
                   /p:DotnetPublishSdkAssetsBlobFeedUrl=$(_PublishBlobFeedUrl)
-                  /p:DotNetPublishSdkAssetsBlobFeedKey=$(dotnetcli-storage-key)
                   /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
                   /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
                   /p:PB_PublishType=blob
                   /p:DotnetPublishChecksumsBlobFeedUrl=$(_PublishChecksumsBlobFeedUrl)
-                  /p:DotNetPublishChecksumsBlobFeedKey=$(dotnetclichecksums-storage-key)
                   /p:DotNetSignType=real
                   /p:TeamName=$(_TeamName)
+                  /p:DotNetPublishUsingPipelines=$(_PublishUsingPipelines)
 
     steps:
     - checkout: self

eng/docker/alpine.3.6/Dockerfile

@@ -22,5 +22,6 @@ RUN chmod -R a+rwx /home
 USER ${USER_ID}
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}
 

eng/docker/centos/Dockerfile

@@ -22,4 +22,5 @@ RUN chmod -R 755 /usr/bin/sudo
 USER ${USER_ID}
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}

eng/docker/debian/Dockerfile

@@ -28,4 +28,5 @@ RUN chmod -R 755 /usr/lib/sudo
 USER ${USER_ID}
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}

eng/docker/fedora.23/Dockerfile

@@ -27,4 +27,5 @@ RUN chmod -R a+rwx /home
 USER ${USER_ID}
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}

eng/docker/fedora.24/Dockerfile

@@ -25,4 +25,5 @@ RUN chmod -R a+rwx /home
 USER ${USER_ID} 
  
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR 
+WORKDIR ${WORK_DIR}

eng/docker/fedora.27/Dockerfile

@@ -29,4 +29,5 @@ RUN chmod -R a+rwx /home
 USER ${USER_ID} 
  
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR 
+WORKDIR ${WORK_DIR}

eng/docker/fedora.28/Dockerfile

@@ -26,4 +26,5 @@ RUN chmod -R a+rwx /home
 USER ${USER_ID} 
  
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR 
+WORKDIR ${WORK_DIR}

eng/docker/fedora.29/Dockerfile

@@ -26,4 +26,5 @@ RUN chmod -R a+rwx /home
 USER ${USER_ID} 
  
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR 
+WORKDIR ${WORK_DIR}

eng/docker/opensuse.13.2/Dockerfile

@@ -22,4 +22,5 @@ RUN chmod -R 755 /usr/lib/sudo
 USER ${USER_ID}
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}

eng/docker/opensuse.42.1/Dockerfile

@@ -47,4 +47,5 @@ RUN chmod -R 755 /usr/lib/sudo
 USER ${USER_ID}
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}

eng/docker/opensuse.42.3/Dockerfile

@@ -46,4 +46,5 @@ RUN chmod -R 755 /usr/lib/sudo
 USER ${USER_ID}
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}

eng/docker/rhel.6/Dockerfile

@@ -27,5 +27,6 @@ USER ${USER_ID}
 ENV LD_LIBRARY_PATH /usr/local/lib
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}
 

eng/docker/rhel/Dockerfile

@@ -20,4 +20,5 @@ RUN chown root:root /usr/bin/sudo && chmod 4755 /usr/bin/sudo
 USER ${USER_ID}
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}

eng/docker/ubuntu.14.04/Dockerfile

@@ -60,4 +60,5 @@ RUN chmod -R 755 /usr/lib/sudo
 USER ${USER_ID}
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}

eng/docker/ubuntu.16.04/Dockerfile

@@ -53,4 +53,5 @@ RUN chmod -R 755 /usr/lib/sudo
 USER ${USER_ID}
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}

eng/docker/ubuntu.16.10/Dockerfile

@@ -60,4 +60,5 @@ RUN chmod -R 755 /usr/lib/sudo
 USER ${USER_ID}
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}

eng/docker/ubuntu.18.04/Dockerfile

@@ -28,4 +28,5 @@ RUN chmod -R 755 /usr/lib/sudo
 USER ${USER_ID}
 
 # Set working directory 
-WORKDIR /opt/code
+ARG WORK_DIR
+WORKDIR ${WORK_DIR}

eng/dockerrun.ps1

@@ -19,7 +19,7 @@ Write-Host "Additional args: $additionalArgs"
 
 $dockerFile = Resolve-Path (Join-Path $RepoRoot "eng\docker\$dockerImageName")
 
-docker build --build-arg USER_ID=1000 -t "$dockerContainerTag" $dockerFile
+docker build --build-arg WORK_DIR=$RepoRoot --build-arg USER_ID=1000 -t "$dockerContainerTag" $dockerFile
 
 $interactiveFlag = "-i"
 if ($noninteractive)
@@ -31,7 +31,7 @@ if ($noninteractive)
 
 docker run $interactiveFlag -t --rm --sig-proxy=true `
   --name "$dockerContainerName" `
-  -v "${RepoRoot}:/opt/code" `
+  -v "${RepoRoot}:${RepoRoot}" `
   -e DOTNET_CORESDK_IGNORE_TAR_EXIT_CODE=1 `
   -e CHANNEL `
   -e DOTNET_BUILD_SKIP_CROSSGEN `
@@ -57,4 +57,4 @@ docker run $interactiveFlag -t --rm --sig-proxy=true `
   -e EXTERNALRESTORESOURCES `
   -e ARCADE_PARTITION="${dockerImageName}" `
   $dockerContainerTag `
-  /opt/code/run-build.sh @additionalArgs
+  ${RepoRoot}/run-build.sh @additionalArgs

eng/dockerrun.sh

@@ -39,7 +39,7 @@ while [[ $# > 0 ]]; do
             echo "Options:"
             echo "  <Dockerfile>    The path to the Dockerfile to use to create the build container"
             echo "  <ImageName>     The name of an existing Dockerfile folder under eng/docker to use as the Dockerfile"
-            echo "  <Command>  The command to run once inside the container (/opt/code is mapped to the repo root; defaults to nothing, which runs the default shell)"
+            echo "  <Command>  The command to run once inside the container (repo root is mapped to DOCKER_HOST_SHARE_DIR; defaults to nothing, which runs the default shell)"
             exit 0
             ;;
         *)
@@ -107,7 +107,7 @@ fi
 
 # Build the docker container (will be fast if it is already built)
 echo "Building Docker Container using Dockerfile: $DOCKERFILE"
-docker build --build-arg USER_ID=$(id -u) -t $DOTNET_BUILD_CONTAINER_TAG $DOCKERFILE
+docker build --build-arg WORK_DIR=$DOCKER_HOST_SHARE_DIR --build-arg USER_ID=$(id -u) -t $DOTNET_BUILD_CONTAINER_TAG $DOCKERFILE
 
 # Run the build in the container
 echo "Launching build in Docker Container"
@@ -118,7 +118,7 @@ echo "Using code from: $DOCKER_HOST_SHARE_DIR"
 # Note: passwords/keys should not be passed in the environment
 docker run $INTERACTIVE -t --rm --sig-proxy=true \
     --name $DOTNET_BUILD_CONTAINER_NAME \
-    -v $DOCKER_HOST_SHARE_DIR:/opt/code \
+    -v $DOCKER_HOST_SHARE_DIR:$DOCKER_HOST_SHARE_DIR \
     -e CHANNEL \
     -e DOTNET_BUILD_SKIP_CROSSGEN \
     -e PUBLISH_TO_AZURE_BLOB \