Publish to bar (#328)

* Enable publishing to BAR

* Update feed package version

* var group not available in public

* move asset manifest parameter to variable

* Fix matrix

.vsts-ci.yml

@@ -2,34 +2,27 @@ trigger:
 - master
 
 variables:
-  _TeamName: Roslyn-Project-System
+- name: _PublishBlobFeedUrl
-  ${{ if eq(variables['System.TeamProject'], 'public') }}:
+  value: https://dotnetcli.blob.core.windows.net/dotnet/index.json
-    PB_PublishBlobFeedKey:
+- name: _PublishChecksumsBlobFeedUrl
-    PB_PublishBlobFeedUrl:
+  value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json
-    PB_PublishChecksumsBlobFeedUrl:
+- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-    _DotNetPublishToBlobFeed: false
+  - group: DotNet-DotNetCli-Storage
-    _PublishType: none
-    _SignType: test
-  ${{ if ne(variables['System.TeamProject'], 'public') }}:
-    PB_PublishBlobFeedUrl: https://dotnetcli.blob.core.windows.net/dotnet/index.json
-    PB_PublishChecksumsBlobFeedUrl: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json
-    _DotNetPublishToBlobFeed: true
-    _PublishType: blob
-    _SignType: real
 
 jobs:
 - template: /eng/build.yml
   parameters:
     agentOs: Windows_NT
-    queue:
+    pool:
-      ${{ if eq(variables['System.TeamProject'], 'public') }}:
+      ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
         name: dotnet-external-temp
-      ${{ if ne(variables['System.TeamProject'], 'public') }}:
+      ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
         name: dotnet-internal-temp
-      timeoutInMinutes: 180
+    timeoutInMinutes: 180
-      parallel: 99
+    strategy:
       matrix:
-        ${{ if eq(variables['System.TeamProject'], 'public') }}:
+        # Public-only builds
+        ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
           Build_Debug_x86:
             _BuildConfig: Debug
             _BuildArchitecture: x86
@@ -45,33 +38,38 @@ jobs:
             _BuildArchitecture: x64
             _DOTNET_CLI_UI_LANGUAGE: ''
             _DropSuffix: true
-        ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        # Internal-only builds
+        ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
           Build_Release_x86:
             _BuildConfig: Release
             _BuildArchitecture: x86
             _DOTNET_CLI_UI_LANGUAGE: ''
             _DropSuffix: ''
+            _PublishBuildAssetManifest: true
+        # Always run builds
         Build_Release_x64:
           _BuildConfig: Release
           _BuildArchitecture: x64
           _DOTNET_CLI_UI_LANGUAGE: ''
           _DropSuffix: ''
+          _PublishBuildAssetManifest: true
         Build_Release_arm:
           _BuildConfig: Release
           _BuildArchitecture: arm
           _DOTNET_CLI_UI_LANGUAGE: ''
           _DropSuffix: ''
           _AdditionalBuildParameters: '/p:DisableSourceLink=true'
+          _PublishBuildAssetManifest: true
 
 - template: /eng/build.yml
   parameters:
     agentOs: Linux
-    queue:
+    pool:
       name: Hosted Ubuntu 1604
-      timeoutInMinutes: 180
+    timeoutInMinutes: 180
-      parallel: 99
+    strategy:
       matrix:
-        ${{ if eq(variables['System.TeamProject'], 'public') }}:
+        ${{ if or(eq(variables['System.TeamProject'], 'public'), in(variables['Build.Reason'], 'PullRequest')) }}:
           Build_Ubuntu_14_04_Release_x64:
             _BuildConfig: Release
             _DockerParameter: '--docker ubuntu.14.04'
@@ -170,7 +168,23 @@ jobs:
             _RuntimeIdentifier: ''
             _BuildArchitecture: 'arm'
             _DropSuffix: true
-        ${{ if ne(variables['System.TeamProject'], 'public') }}:
+        ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+          Build_Arm_Release:
+            _BuildConfig: Release
+            _DockerParameter: ''
+            _LinuxPortable: '--linux-portable'
+            _RuntimeIdentifier: ''
+            _BuildArchitecture: 'arm'
+            _DropSuffix: ''
+            _PublishBuildAssetManifest: true
+          Build_Arm64_Release:
+            _BuildConfig: Release
+            _DockerParameter: ''
+            _LinuxPortable: '--linux-portable'
+            _RuntimeIdentifier: ''
+            _BuildArchitecture: 'arm64'
+            _DropSuffix: ''
+            _PublishBuildAssetManifest: true
           Build_Rhel_6_Release_x64:
             _BuildConfig: Release
             _DockerParameter: '--docker rhel.6'
@@ -179,20 +193,6 @@ jobs:
             _BuildArchitecture: 'x64'
             _DropSuffix: ''
             _AdditionalBuildParameters: '/p:DisableSourceLink=true'
-          Build_Arm_Release:
-            _BuildConfig: Release
-            _DockerParameter: ''
-            _LinuxPortable: '--linux-portable'
-            _RuntimeIdentifier: ''
-            _BuildArchitecture: 'arm'
-            _DropSuffix: ''
-          Build_Arm64_Release:
-            _BuildConfig: Release
-            _DockerParameter: ''
-            _LinuxPortable: '--linux-portable'
-            _RuntimeIdentifier: ''
-            _BuildArchitecture: 'arm64'
-            _DropSuffix: ''
           Build_Linux_musl_Release_x64:
             _BuildConfig: Release
             _DockerParameter: '--docker alpine.3.6'
@@ -227,16 +227,19 @@ jobs:
           _RuntimeIdentifier: ''
           _BuildArchitecture: 'x64'
           _DropSuffix: ''
+          _PublishBuildAssetManifest: true
 
 - template: /eng/build.yml
   parameters:
     agentOs: Darwin
-    queue: 
+    pool: 
       name: Hosted macOS
-      timeoutInMinutes: 180
+    timeoutInMinutes: 180
+    strategy:
       matrix:
         Build_Release:
           _BuildConfig: Release
+          _PublishBuildAssetManifest: true
 
 # https://github.com/dotnet/core-sdk/issues/248
 # - template: /eng/build.yml
@@ -259,29 +262,23 @@ jobs:
     pool:
       name: Hosted VS2017
     condition: succeeded()
+    variables:
+      - group: DotNet-DotNetCli-Storage
     steps:
-      - task: AzureKeyVault@1
-        inputs:
-          azureSubscription: 'DotNet-Engineering-Services_KeyVault'
-          KeyVaultName: EngKeyVault
-          SecretsFilter: 'dotnetcli-storage-key,dotnetclichecksums-storage-key'
-        condition: succeeded()
-
       - script: eng/CopyToLatest.cmd
-                /p:DotnetPublishSdkAssetsBlobFeedUrl=$(PB_PublishBlobFeedUrl)
+                /p:DotnetPublishSdkAssetsBlobFeedUrl=$(_PublishBlobFeedUrl)
                 /p:DotNetPublishSdkAssetsBlobFeedKey=$(dotnetcli-storage-key)
-                /p:DotnetPublishChecksumsBlobFeedUrl=$(PB_PublishChecksumsBlobFeedUrl)
+                /p:DotnetPublishChecksumsBlobFeedUrl=$(_PublishChecksumsBlobFeedUrl)
                 /p:DotNetPublishChecksumsBlobFeedKey=$(dotnetclichecksums-storage-key)
         displayName: Copy to latest
 
-# - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-#   - template: /eng/common/templates/phases/publish-build-assets.yml
+  - template: /eng/common/templates/job/publish-build-assets.yml
-#     parameters:
+    parameters:
-#       dependsOn:
+      dependsOn:
-#         - Windows_NT
+        - Windows_NT
-#         - Linux
+        - Linux
-#         - Darwin
+        - Darwin
-#         - Copy_SDK_To_Latest
+      enablePublishBuildArtifacts: true
-#       queue:
+      pool:
-#         name: Hosted VS2017
+        vmImage: vs2017-win2016
-#         timeoutInMinutes: 180

eng/Publishing.props

@@ -31,8 +31,9 @@
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.rpm" />
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.msi" />
     <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.nupkg" />
-    <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.svg" Condition=" '$(PublishBinariesAndBadge)' == 'true' " />
+    <SdkAssetsToPublish Include="$(ArtifactsShippingPackagesDir)*.svg" Condition=" '$(PublishBinariesAndBadge)' == 'true' ">
-
+      <ExcludeFromManifest>true</ExcludeFromManifest>
+    </SdkAssetsToPublish>
     <CheckSumsToPublish Include="$(ArtifactsShippingPackagesDir)*.sha" />
     <CheckSumsToPublish Remove="$(ArtifactsShippingPackagesDir)*.zip.sha" Condition=" '$(PublishBinariesAndBadge)' == 'false' "/>
     <CheckSumsToPublish Remove="$(ArtifactsShippingPackagesDir)*.tar.gz.sha" Condition=" '$(PublishBinariesAndBadge)' == 'false' "/>
@@ -64,7 +65,7 @@
       ManifestBuildId="$(BUILD_BUILDNUMBER)"
       ManifestCommit="$(BUILD_SOURCEVERSION)"
       AssetManifestPath="$(AssetManifestFilePath)"
-      PublishFlatContainer="true"/>
+      PublishFlatContainer="true" />
 
     <PushToBlobFeed
       ExpectedFeedUrl="$(ChecksumsFeedUrl)"

eng/Versions.props

@@ -50,7 +50,7 @@
   </PropertyGroup>
   <!-- infrastructure and test only dependencies -->
   <PropertyGroup>
-    <BuildTasksFeedToolVersion>2.1.0-prerelease-02430-04</BuildTasksFeedToolVersion>
+    <BuildTasksFeedToolVersion>2.2.0-beta.19072.10</BuildTasksFeedToolVersion>
     <VersionToolsVersion>$(BuildTasksFeedToolVersion)</VersionToolsVersion>
     <DotnetDebToolVersion>2.0.0</DotnetDebToolVersion>
     <MicrosoftNETTestSdkVersion>15.8.0</MicrosoftNETTestSdkVersion>

eng/build.yml

@@ -1,102 +1,88 @@
 parameters:
+  # Agent OS identifier and used as job name
   agentOs: ''
-  queue: {}
+
+  # Agent pool
+  pool: {}
+
+  # Additional variables
   variables: {}
 
+  # Build strategy - matrix
+  strategy: {}
+
+  # Job timeout
+  timeoutInMinutes: 180
+
 phases:
-- template: /eng/common/templates/phases/base.yml
+- template: /eng/common/templates/job/job.yml
   parameters:
     name: ${{ parameters.agentOs }}
+    ${{ if eq(parameters.agentOs, 'Windows_NT') }}:
+      enableMicrobuild: true
+    enablePublishBuildArtifacts: true
+    enablePublishBuildAssets: true
+    enablePublishTestResults: true
     enableTelemetry: true
-    enableMicrobuild: true
+    helixRepo: dotnet/core-sdk
-    publicBuildReasons: PullRequest
+    pool: ${{ parameters.pool }}
-    queue: ${{ parameters.queue }}
+    ${{ if ne(parameters.strategy, '') }}:
+      strategy: ${{ parameters.strategy }}
+
     variables: 
-      ${{ insert }}: ${{ parameters.variables }}
+      - _TeamName: Roslyn-Project-System
-      _AgentOSName: ${{ parameters.agentOs }}
+      - _BuildArgs: ''
-      _HelixType: build/product
+
-      _HelixBuildConfig: $(_BuildConfig)
+      - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-      BuildConfig: $(_BuildConfig)
+        - group: DotNet-Symbol-Server-PATs
-      BuildArchitecture: $(_BuildArchitecture)
+        - group: DotNet-HelixApi-Access
-      DockerParameter: $(_DockerParameter)
+        - _DotNetPublishToBlobFeed: true
-      LinuxPortable: $(_LinuxPortable)
+        - _BuildArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
-      RuntimeId: $(_RuntimeIdentifier)
+                  /p:DotNetPublishToBlobFeed=$(_DotNetPublishToBlobFeed)
-      AdditionalBuildParameters: $(_AdditionalBuildParameters)
+                  /p:DotnetPublishSdkAssetsBlobFeedUrl=$(_PublishBlobFeedUrl)
-      ${{ if eq(variables['System.TeamProject'], 'public') }}:
-        _HelixSource: pr/dotnet/core-sdk/$(Build.SourceBranch)
-        _OfficialBuildIdArgs: ''
-        _PublishArgs: ''
-        _SignArgs: ''
-      ${{ if ne(variables['System.TeamProject'], 'public') }}:
-        _HelixSource: official/dotnet/core-sdk/$(Build.SourceBranch)
-        _OfficialBuildIdArgs: /p:OfficialBuildId=$(BUILD.BUILDNUMBER)
-        _PublishArgs: /p:DotNetPublishToBlobFeed=$(_DotNetPublishToBlobFeed)
-                  /p:DotnetPublishSdkAssetsBlobFeedUrl=$(PB_PublishBlobFeedUrl)
                   /p:DotNetPublishSdkAssetsBlobFeedKey=$(dotnetcli-storage-key)
                   /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
                   /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
-                  /p:PB_PublishType=$(_PublishType)
+                  /p:PB_PublishType=blob
-                  /p:DotnetPublishChecksumsBlobFeedUrl=$(PB_PublishChecksumsBlobFeedUrl)
+                  /p:DotnetPublishChecksumsBlobFeedUrl=$(_PublishChecksumsBlobFeedUrl)
                   /p:DotNetPublishChecksumsBlobFeedKey=$(dotnetclichecksums-storage-key)
-        _SignArgs: /p:DotNetSignType=$(_SignType) /p:TeamName=$(_TeamName)
+                  /p:AssetManifestFilePath=$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest/$(Agent.OS)_$(_BuildArchitecture).xml
+                  /p:DotNetSignType=real
+                  /p:TeamName=$(_TeamName)
 
     steps:
-    - script: 'eng\setbuildinfo.bat $(BuildArchitecture) $(BuildConfig)'
+    - checkout: self
-      condition: eq(variables['_AgentOSName'], 'Windows_NT')
+      clean: true
-      displayName: Set test info (Windows)
-    - script: 'eng/setbuildinfo.sh $(BuildArchitecture) $(BuildConfig)'
-      condition: ne(variables['_AgentOSName'], 'Windows_NT')
-      displayName: Set test info (Non-Windows)
-    - script: 'echo TestParameter: $(TestParameter)'
-      displayName: Show TestParameter
-    - script: 'echo RunTests: $(RunTests)'
-      displayName: Show RunTests
-    - script: 'echo AdditionalBuildParameters: $(AdditionalBuildParameters)'
-      displayName: Show AdditionalBuildParameters
-
-    - ${{ if ne(variables['System.TeamProject'], 'public') }}:
-      - task: AzureKeyVault@1
-        inputs:
-          azureSubscription: 'DotNet-Engineering-Services_KeyVault'
-          KeyVaultName: EngKeyVault
-          SecretsFilter: 'dotnetcli-storage-key,dotnetclichecksums-storage-key,microsoft-symbol-server-pat,symweb-symbol-server-pat'
-        condition: succeeded()
-      - task: AzureKeyVault@1
-        inputs:
-          azureSubscription: 'HelixProd_KeyVault'
-          KeyVaultName: HelixProdKV
-          SecretsFilter: 'HelixApiAccessToken'
-        condition: always()
-
     - ${{ if eq(parameters.agentOs, 'Windows_NT') }}:
+      - script: 'eng\setbuildinfo.bat $(_BuildArchitecture) $(_BuildConfig)'
+        displayName: Set test info (Windows)
       - script: build.cmd
                   $(TestParameter)
                   -pack -publish -ci
-                  -Configuration $(BuildConfig)
+                  -Configuration $(_BuildConfig)
-                  -Architecture $(BuildArchitecture)
+                  -Architecture $(_BuildArchitecture)
-                  $(_PublishArgs)
+                  $(_BuildArgs)
-                  $(_SignArgs)
+                  $(_AdditionalBuildParameters)
-                  $(_OfficialBuildIdArgs)
-                  $(AdditionalBuildParameters)
         displayName: Build
         env:
           DOTNET_CLI_UI_LANGUAGE: $(_DOTNET_CLI_UI_LANGUAGE)
           DropSuffix: $(_DropSuffix)
 
+    - ${{ if ne(parameters.agentOs, 'Windows_NT') }}:
+      - script: 'eng/setbuildinfo.sh $(_BuildArchitecture) $(_BuildConfig)'
+        displayName: Set test info (Non-Windows)
     - ${{ if eq(parameters.agentOs, 'Linux') }}:
       - script: ./build.sh
                   $(TestParameter)
                   --pack --publish --ci
                   --noprettyprint
-                  --configuration $(BuildConfig)
+                  --configuration $(_BuildConfig)
-                  $(DockerParameter)
+                  $(_DockerParameter)
-                  --architecture $(BuildArchitecture)
+                  --architecture $(_BuildArchitecture)
-                  $(LinuxPortable)
+                  $(_LinuxPortable)
-                  $(RuntimeId)
+                  $(_RuntimeIdentifier)
-                  $(_PublishArgs)
+                  $(_BuildArgs)
-                  $(_SignArgs)
+                  $(_AdditionalBuildParameters)
-                  $(_OfficialBuildIdArgs)
-                  $(AdditionalBuildParameters)
         displayName: Build
         env:
           DropSuffix: $(_DropSuffix)
@@ -106,38 +92,24 @@ phases:
                   $(TestParameter)
                   --pack --publish --ci
                   --noprettyprint
-                  --configuration $(BuildConfig)
+                  --configuration $(_BuildConfig)
-                  $(_PublishArgs)
+                  $(_BuildArgs)
-                  $(_SignArgs)
+                  $(_AdditionalBuildParameters)
-                  $(_OfficialBuildIdArgs)
-                  $(AdditionalBuildParameters)
         displayName: Build
 
-    - task: PublishTestResults@1
+    - ${{ if and(eq(parameters.enablePublishBuildAssets, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
-      displayName: Publish Test Results
+      - task: CopyFiles@2
-      inputs:
+        displayName: Gather Asset Manifests
-        testRunner: XUnit
+        inputs:
-        testResultsFiles: 'artifacts/TestResults/$(_BuildConfig)/*.xml'
+          SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
-        testRunTitle: '$(_AgentOSName)_$(Agent.JobName)'
+          TargetFolder: '$(Build.StagingDirectory)/AssetManifests'
-        platform: '$(BuildPlatform)'
+        continueOnError: ${{ parameters.continueOnError }}
-        configuration: '$(_BuildConfig)'
+        condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'), eq(variables['_PublishBuildAssetManifest'], 'true'))
-      condition: and(succeededOrFailed(), ne(variables['PB_SkipTests'], 'true'), ne(variables['RunTests'], 'false'))
+      - task: PublishBuildArtifacts@1
-    - task: CopyFiles@2
+        displayName: Push Asset Manifests
-      displayName: Gather Logs
+        inputs:
-      inputs:
+          PathtoPublish: '$(Build.StagingDirectory)/AssetManifests'
-        SourceFolder: '$(Build.SourcesDirectory)'
+          PublishLocation: Container
-        Contents: |
+          ArtifactName: AssetManifests
-         artifacts/log/$(_BuildConfig)/**/*
+        continueOnError: ${{ parameters.continueOnError }}
-         artifacts/TestResults/$(_BuildConfig)/**/*
+        condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'), eq(variables['_PublishBuildAssetManifest'], 'true'))
-        TargetFolder: '$(Build.ArtifactStagingDirectory)'
-      continueOnError: true
-      condition: always()
-
-    - task: PublishBuildArtifacts@1
-      displayName: Publish Logs to VSTS
-      inputs:
-        PathtoPublish: '$(Build.ArtifactStagingDirectory)'
-        ArtifactName: '$(_AgentOSName)_$(Agent.JobName)_$(Build.BuildNumber)'
-        publishLocation: Container
-      continueOnError: true
-      condition: always()