ci: bake appveyor images automatically, run sync on depshash change (#35396)

* chore: update yml formatting for parser

* ci: bake appveyor images automatically, run sync on depshash change

* chore: clean up .yml files

* chore: bump to e-110.0.5415.0

.github/workflows/update_appveyor_image.yml

@@ -0,0 +1,62 @@
+name: Update AppVeyor Image
+
+# Run chron daily Mon-Fri
+on:
+  schedule:
+    - cron: '0 8 * * 1-5' # runs 8:00 every business day (see https://crontab.guru)
+
+permissions:
+  contents: write
+  pull-requests: write
+
+jobs:
+  bake-appveyor-image:
+    name: Bake AppVeyor Image
+    permissions:
+      contents: write
+      pull-requests: write  # to create a new PR with updated Appveyor images
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+    - name: Yarn install
+      run: |
+        node script/yarn.js install --frozen-lockfile
+    - name: Set Repo for Commit
+      run: git config --global --add safe.directory $GITHUB_WORKSPACE
+    - name: Check AppVeyor Image
+      env:
+        APPVEYOR_TOKEN: ${{ secrets.APPVEYOR_TOKEN }}
+      run: |
+        node ./script/prepare-appveyor
+        if [ -f ./image_version.txt ]; then
+          echo "APPVEYOR_IMAGE_VERSION="$(cat image_version.txt)"" >> $GITHUB_ENV
+          rm image_version.txt
+        fi
+    - name: (Optionally) Update Appveyor Image
+      if: ${{ env.APPVEYOR_IMAGE_VERSION }}
+      uses: mikefarah/yq@v4.27.2
+      with:
+        cmd: yq '.image = "${{ env.APPVEYOR_IMAGE_VERSION }}"' "appveyor.yml" > "appveyor2.yml"
+    - name: (Optionally) Generate Commit Diff
+      if: ${{ env.APPVEYOR_IMAGE_VERSION }}
+      run: |
+        diff -w -B appveyor.yml appveyor2.yml > appveyor.diff || true
+        patch -f appveyor.yml < appveyor.diff
+        rm appveyor2.yml appveyor.diff
+    - name: (Optionally) Commit and Pull Request
+      if: ${{ env.APPVEYOR_IMAGE_VERSION }}
+      uses: peter-evans/create-pull-request@v4
+      with:
+        token: ${{ secrets.ACTIONS_GITHUB_TOKEN }}
+        commit-message: 'build: update appveyor image to latest version'
+        committer: GitHub <noreply@github.com>
+        author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+        signoff: false
+        branch: bump-appveyor-image
+        delete-branch: true
+        title: 'build: update appveyor image to latest version'
+        body: |
+          This PR updates appveyor.yml to the latest baked image, ${{ env.APPVEYOR_IMAGE_VERSION }}.

appveyor-bake.yml

@@ -0,0 +1,52 @@
+# The config is used to bake appveyor images, not for running CI jobs.
+# The config expects the following environment variables to be set:
+#  - "APPVEYOR_BAKE_IMAGE" e.g. 'electron-99.0.4767.0'. Name of the image to be baked.
+#      Typically named after the Chromium version on which the image is built.
+#      This can be set dynamically in the prepare-appveyor script.
+
+version: 1.0.{build}
+build_cloud: electronhq-16-core
+image: Windows_Default_Appveyor
+environment:
+  GIT_CACHE_PATH: C:\Users\appveyor\libcc_cache
+  ELECTRON_OUT_DIR: Default
+  ELECTRON_ENABLE_STACK_DUMPING: 1
+  MOCHA_REPORTER: mocha-multi-reporters
+  MOCHA_MULTI_REPORTERS: mocha-appveyor-reporter, tap
+  GOMA_FALLBACK_ON_AUTH_FAILURE: true
+  DEPOT_TOOLS_WIN_TOOLCHAIN: 0
+  PYTHONIOENCODING: UTF-8
+build_script:
+  - ps: Resize-Partition -DriveLetter C -Size (256GB) # ensure initial partition size
+  - ps: Get-Partition -DriveLetter C
+  - git config --global core.longpaths true
+  - cd ..
+  - mkdir src
+  - ps: git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git  
+  - ps: $env:PATH="$pwd\depot_tools;$env:PATH"  
+  - update_depot_tools.bat
+  - ps: Move-Item $env:APPVEYOR_BUILD_FOLDER -Destination src\electron
+  - src\electron\script\setup-win-for-dev.bat
+  - >-
+      gclient config
+      --name "src\electron"
+      --unmanaged
+      %GCLIENT_EXTRA_ARGS%
+      "https://github.com/electron/electron"
+  - ps: cd src\electron
+  - ps: node script\generate-deps-hash.js
+  - ps: $depshash = Get-Content .\.depshash -Raw
+  - ps: Copy-Item -path .\.depshash -destination ..\.depshash
+  - ps: cd ..\..
+  - gclient sync --with_branch_heads --with_tags --nohooks
+  - ps: regsvr32 /s "C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\DIA SDK\bin\amd64\msdia140.dll"
+on_image_bake:
+  - ps: >-
+      echo "Baking image: $env:APPVEYOR_BAKE_IMAGE at dir $PWD"
+  - ps: Remove-Item -Recurse -Force $pwd\depot_tools
+  - ps: Remove-Item -Recurse -Force $pwd\src\electron
+# Uncomment these lines to enable RDP
+#on_finish:
+#  - ps: >-
+#       $env:APPVEYOR_RDP_PASSWORD = "electron"
+#       $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))

appveyor.yml

@@ -24,21 +24,22 @@
 # https://www.appveyor.com/docs/build-configuration/#custom-environment-variables
 
 version: 1.0.{build}
-build_cloud: electron-16-core
+build_cloud: electronhq-16-core
-image: vs2019bt-16.16.11
+image: e-110.0.5415.0
 environment:
-  GIT_CACHE_PATH: C:\Users\electron\libcc_cache
+  GIT_CACHE_PATH: C:\Users\appveyor\libcc_cache
   ELECTRON_OUT_DIR: Default
   ELECTRON_ENABLE_STACK_DUMPING: 1
   ELECTRON_ALSO_LOG_TO_STDERR: 1
   MOCHA_REPORTER: mocha-multi-reporters
   MOCHA_MULTI_REPORTERS: mocha-appveyor-reporter, tap
   GOMA_FALLBACK_ON_AUTH_FAILURE: true
+  DEPOT_TOOLS_WIN_TOOLCHAIN: 0
+  PYTHONIOENCODING: UTF-8
 
   matrix:
 
     - job_name: Build
-
     - job_name: Test
       job_depends_on: Build
 
@@ -50,8 +51,7 @@ matrix:
 
 for:
 
-  -
+  - matrix:
-    matrix:
       only:
         - job_name: Build
 
@@ -66,13 +66,26 @@ for:
           node script/yarn.js install --frozen-lockfile
           node script/doc-only-change.js --prNumber=$env:APPVEYOR_PULL_REQUEST_NUMBER --prBranch=$env:APPVEYOR_REPO_BRANCH
           if ($LASTEXITCODE -eq 0) {
-          Write-warning "Skipping tests for doc only change"; Exit-AppveyorBuild
+            Write-warning "Skipping build for doc only change"; Exit-AppveyorBuild
           }
           $global:LASTEXITCODE = 0
       - cd ..
       - ps: Write-Host "Building $env:GN_CONFIG build"
       - git config --global core.longpaths true
-    - update_depot_tools.bat
+      - ps: >-
+          if (Test-Path -Path "$pwd\depot_tools") {
+            Remove-Item -Recurse -Force $pwd\depot_tools
+          }
+      - ps: >-
+          if (Test-Path -Path "$pwd\build-tools") {
+            Remove-Item -Recurse -Force $pwd\build-tools
+          }
+      - ps: git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git
+      - ps: $env:PATH="$pwd\depot_tools;$env:PATH"
+      - ps: >-
+          if (Test-Path -Path "$pwd\src\electron") {
+            Remove-Item -Recurse -Force $pwd\src\electron
+          }
       - ps: >-
           if (Test-Path 'env:RAW_GOMA_AUTH') {
             $env:GOMA_OAUTH2_CONFIG_FILE = "$pwd\.goma_oauth2_config"
@@ -103,70 +116,19 @@ for:
           if ($env:GN_CONFIG -ne 'release') {
             $env:NINJA_STATUS="[%r processes, %f/%t @ %o/s : %es] "
           }
-    - >-
+      - gclient config --name "src\electron" --unmanaged %GCLIENT_EXTRA_ARGS% "https://github.com/electron/electron"
-        gclient config
+      # Patches are applied in the image bake. Check depshash to see if patches have changed. 
-        --name "src\electron"
+      - ps: $env:RUN_GCLIENT_SYNC="false"
-        --unmanaged
+      - ps: $depshash_baked = Get-Content .\src\.depshash -Raw
-        %GCLIENT_EXTRA_ARGS%
+      - ps: cd src\electron
-        "https://github.com/electron/electron"
+      - ps: node script\generate-deps-hash.js
+      - ps: $depshash = Get-Content .\.depshash -Raw
+      - ps: cd ..\..
       - ps: >-
-        if ($env:GN_CONFIG -eq 'release') {
+          if ($depshash_baked -ne $depshash) {
             $env:RUN_GCLIENT_SYNC="true"
-        } else {
-          cd src\electron
-          node script\generate-deps-hash.js
-          $depshash = Get-Content .\.depshash -Raw 
-          $zipfile = "Z:\$depshash.7z"
-          cd ..\..
-          if (Test-Path -Path $zipfile) {
-            # file exists, unzip and then gclient sync
-            7z x -y $zipfile -mmt=14 -aoa
-            if (-not (Test-Path -Path "src\buildtools")) {
-              # the zip file must be corrupt - resync
-              $env:RUN_GCLIENT_SYNC="true"
-              if ($env:TARGET_ARCH -ne 'ia32') {
-                # only save on x64/woa to avoid contention saving
-                $env:SAVE_GCLIENT_SRC="true"
-              }
-            } else {
-              # update angle
-              cd src\third_party\angle
-              git remote set-url origin  https://chromium.googlesource.com/angle/angle.git
-              git fetch
-              cd ..\..\..
-            }
-          } else {    
-            # file does not exist, gclient sync, then zip
-            $env:RUN_GCLIENT_SYNC="true"
-            if ($env:TARGET_ARCH -ne 'ia32') {
-              # only save on x64/woa to avoid contention saving
-              $env:SAVE_GCLIENT_SRC="true"
-            }
-          }
-        }
-    - if "%RUN_GCLIENT_SYNC%"=="true" ( gclient sync )
-    - ps: >-
-        if ($env:SAVE_GCLIENT_SRC -eq 'true') {
-          # archive current source for future use 
-          # only run on x64/woa to avoid contention saving
-          $(7z a $zipfile src -xr!android_webview -xr!electron -xr'!*\.git' -xr!third_party\blink\web_tests -xr!third_party\blink\perf_tests -slp -t7z -mmt=30)
-          if ($LASTEXITCODE -ne 0) {
-            Write-warning "Could not save source to shared drive; continuing anyway"
-          }
-          # build time generation of file gen/angle/angle_commit.h depends on
-          # third_party/angle/.git
-          # https://chromium-review.googlesource.com/c/angle/angle/+/2074924       
-          $(7z a $zipfile src\third_party\angle\.git)
-          if ($LASTEXITCODE -ne 0) {
-            Write-warning "Failed to add third_party\angle\.git; continuing anyway"
-          }
-          # build time generation of file dawn/common/Version_autogen.h depends on third_party/dawn/.git/HEAD
-          # https://dawn-review.googlesource.com/c/dawn/+/83901        
-          $(7z a $zipfile src\third_party\dawn\.git)
-          if ($LASTEXITCODE -ne 0) {
-            Write-warning "Failed to add third_party\dawn\.git; continuing anyway"
-          }
           }
+      - if "%RUN_GCLIENT_SYNC%"=="true" ( gclient sync --with_branch_heads --with_tags ) else ( gclient runhooks )
       - cd src
       - set BUILD_CONFIG_PATH=//electron/build/args/%GN_CONFIG%.gn
       - gn gen out/Default "--args=import(\"%BUILD_CONFIG_PATH%\") import(\"%GN_GOMA_FILE%\") %GN_EXTRA_ARGS% "
@@ -180,9 +142,8 @@ for:
       - ninja -C out/Default electron:electron_dist_zip
       - ninja -C out/Default shell_browser_ui_unittests
       - gn desc out/Default v8:run_mksnapshot_default args > out/Default/default_mksnapshot_args
-    - ps: >-
       # Remove unused args from mksnapshot_args
-
+      - ps: >-
           Get-Content out/Default/default_mksnapshot_args | Where-Object { -not $_.Contains('--turbo-profiling-input') -And -not $_.Contains('builtins-pgo') } | Set-Content out/Default/mksnapshot_args
       - ninja -C out/Default electron:electron_mksnapshot_zip
       - cd out\Default
@@ -192,6 +153,8 @@ for:
       - ninja -C out/Default electron:electron_chromedriver_zip
       - ninja -C out/Default third_party/electron_node:headers
       - python %LOCAL_GOMA_DIR%\goma_ctl.py stat
+      - ps: >-
+          Get-CimInstance -Namespace root\cimv2 -Class Win32_product | Select vendor, description, @{l='install_location';e='InstallLocation'}, @{l='install_date';e='InstallDate'}, @{l='install_date_2';e='InstallDate2'}, caption, version, name, @{l='sku_number';e='SKUNumber'} | ConvertTo-Json | Out-File -Encoding utf8 -FilePath .\installed_software.json
       - python3 electron/build/profile_toolchain.py --output-json=out/Default/windows_toolchain_profile.json
       - 7z a node_headers.zip out\Default\gen\node_headers
       - ps: >-
@@ -242,9 +205,7 @@ for:
           if ((Test-Path "pdb.zip") -And ($env:GN_CONFIG -ne 'release')) {
             appveyor-retry appveyor PushArtifact pdb.zip
           }
-
+  - matrix:
-  -
-    matrix:
       only:
         - job_name: Test
 
@@ -261,13 +222,13 @@ for:
           node script/yarn.js install --frozen-lockfile
           node script/doc-only-change.js --prNumber=$env:APPVEYOR_PULL_REQUEST_NUMBER --prBranch=$env:APPVEYOR_REPO_BRANCH
           if ($LASTEXITCODE -eq 0) {
-          Write-warning "Skipping tests for doc only change"; Exit-AppveyorBuild
+            Write-warning "Skipping build for doc only change"; Exit-AppveyorBuild
           }
           $global:LASTEXITCODE = 0
+      - cd ..
+      - mkdir out\Default
+      - cd ..
       - ps: |
-        cd ..
-        mkdir out\Default
-        cd ..
           # Download build artifacts
           $apiUrl = 'https://ci.appveyor.com/api'
           $build_info = Invoke-RestMethod -Method Get -Uri "$apiUrl/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/builds/$env:APPVEYOR_BUILD_ID"
@@ -301,15 +262,22 @@ for:
           New-Item .\out\Default\gen\node_headers\Release -Type directory
           Copy-Item -path .\out\Default\electron.lib -destination .\out\Default\gen\node_headers\Release\node.lib
       - cd electron
+      # Explicitly set npm_config_arch because the .env doesn't persist
+      - ps: >-
+          if ($env:TARGET_ARCH -eq 'ia32') {
+            $env:npm_config_arch = "ia32"
+          }
       - echo Running main test suite & node script/yarn test -- --trace-uncaught --runners=main --enable-logging=file --log-file=%cd%\electron.log
       - echo Running native test suite & node script/yarn test -- --trace-uncaught --runners=native --enable-logging=file --log-file=%cd%\electron.log
       - cd ..
-      - echo Verifying non proprietary ffmpeg & python3 electron\script\verify-ffmpeg.py --build-dir out\Default --source-root %cd% --ffmpeg-path out\ffmpeg
+      - echo Verifying non proprietary ffmpeg & python electron\script\verify-ffmpeg.py --build-dir out\Default --source-root %cd% --ffmpeg-path out\ffmpeg
       - echo "About to verify mksnapshot"
-      - echo Verifying mksnapshot & python3 electron\script\verify-mksnapshot.py --build-dir out\Default --source-root %cd%
+      - echo Verifying mksnapshot & python electron\script\verify-mksnapshot.py --build-dir out\Default --source-root %cd%
       - echo "Done verifying mksnapshot"
-      - echo Verifying chromedriver & python3 electron\script\verify-chromedriver.py --build-dir out\Default --source-root %cd%
+      - echo Verifying chromedriver & python electron\script\verify-chromedriver.py --build-dir out\Default --source-root %cd%
       - echo "Done verifying chromedriver"
 
-    on_finish:
+    # Uncomment these lines to enable RDP
+    # on_finish:
+      # - ps: $blockRdp = $true; iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-rdp.ps1'))
       - if exist electron\electron.log ( appveyor-retry appveyor PushArtifact electron\electron.log )

build/profile_toolchain.py

@@ -5,8 +5,6 @@ import sys
 import os
 import optparse
 import json
-import re
-import subprocess
 
 sys.path.append("%s/../../build" % os.path.dirname(os.path.realpath(__file__)))
 
@@ -36,56 +34,10 @@ def calculate_hash(root):
         return CalculateHash('.', None)
 
 def windows_installed_software():
-    powershell_command = [
+    # file_path = os.path.join(os.getcwd(), 'installed_software.json')
-        "Get-CimInstance",
+    # return json.loads(open('installed_software.json').read().decode('utf-8'))
-        "-Namespace",
+    f = open('installed_software.json', encoding='utf-8-sig')
-        "root\cimv2",
+    return json.load(f)
-        "-Class",
-        "Win32_product",
-        "|",
-        "Select",
-        "vendor,",
-        "description,",
-        "@{l='install_location';e='InstallLocation'},",
-        "@{l='install_date';e='InstallDate'},",
-        "@{l='install_date_2';e='InstallDate2'},",
-        "caption,",
-        "version,",
-        "name,",
-        "@{l='sku_number';e='SKUNumber'}",
-        "|",
-        "ConvertTo-Json",
-    ]
-
-    proc = subprocess.Popen(
-        ["powershell.exe", "-Command", "-"],
-        stdin=subprocess.PIPE,
-        stdout=subprocess.PIPE,
-    )
-
-    stdout, _ = proc.communicate(" ".join(powershell_command).encode("utf-8"))
-
-    if proc.returncode != 0:
-        raise RuntimeError("Failed to get list of installed software")
-
-    # On AppVeyor there's other output related to PSReadline,
-    # so grab only the JSON output and ignore everything else
-    json_match = re.match(
-        r".*(\[.*{.*}.*\]).*", stdout.decode("utf-8"), re.DOTALL
-    )
-
-    if not json_match:
-        raise RuntimeError(
-            "Couldn't find JSON output for list of installed software"
-        )
-
-    # Filter out missing keys
-    return list(
-        map(
-            lambda info: {k: info[k] for k in info if info[k]},
-            json.loads(json_match.group(1)),
-        )
-    )
 
 
 def windows_profile():

script/lib/utils.js

@@ -129,6 +129,7 @@ module.exports = {
   getElectronExec,
   getOutDir,
   getAbsoluteElectronExec,
+  handleGitCall,
   ELECTRON_DIR,
   SRC_DIR
 };

script/prepare-appveyor.js

@@ -0,0 +1,219 @@
+if (!process.env.CI) require('dotenv-safe').load();
+
+const assert = require('assert');
+const fs = require('fs');
+const got = require('got');
+const path = require('path');
+const { handleGitCall, ELECTRON_DIR } = require('./lib/utils.js');
+const { Octokit } = require('@octokit/rest');
+const octokit = new Octokit();
+
+const APPVEYOR_IMAGES_URL = 'https://ci.appveyor.com/api/build-clouds';
+const APPVEYOR_JOB_URL = 'https://ci.appveyor.com/api/builds';
+const ROLLER_BRANCH_PATTERN = /^roller\/chromium$/;
+
+const DEFAULT_BUILD_CLOUD_ID = '1598';
+const DEFAULT_BUILD_CLOUD = 'electronhq-16-core';
+const DEFAULT_BAKE_BASE_IMAGE = 'Windows_Default_Appveyor';
+const DEFAULT_BUILD_IMAGE = 'Windows_Default_Appveyor';
+
+const appveyorBakeJob = 'electron-bake-image';
+const appVeyorJobs = {
+  'electron-x64': 'electron-x64-testing',
+  'electron-woa': 'electron-woa-testing',
+  'electron-ia32': 'electron-ia32-testing'
+};
+
+async function makeRequest ({ auth, username, password, url, headers, body, method }) {
+  const clonedHeaders = {
+    ...(headers || {})
+  };
+  if (auth?.bearer) {
+    clonedHeaders.Authorization = `Bearer ${auth.bearer}`;
+  }
+
+  const options = {
+    headers: clonedHeaders,
+    body,
+    method
+  };
+
+  if (username || password) {
+    options.username = username;
+    options.password = password;
+  }
+
+  const response = await got(url, options);
+
+  if (response.statusCode < 200 || response.statusCode >= 300) {
+    console.error('Error: ', `(status ${response.statusCode})`, response.body);
+    throw new Error(`Unexpected status code ${response.statusCode} from ${url}`);
+  }
+  return JSON.parse(response.body);
+}
+
+async function checkAppVeyorImage (options) {
+  const IMAGE_URL = `${APPVEYOR_IMAGES_URL}/${options.cloudId}`;
+  const requestOpts = {
+    url: IMAGE_URL,
+    auth: {
+      bearer: process.env.APPVEYOR_TOKEN
+    },
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    method: 'GET'
+  };
+
+  try {
+    const { settings } = await makeRequest(requestOpts);
+    const { cloudSettings } = settings;
+    return cloudSettings.images.find(image => image.name === `${options.imageVersion}`) || null;
+  } catch (err) {
+    console.log('Could not call AppVeyor: ', err);
+  }
+}
+
+async function getPullRequestId (targetBranch) {
+  const prsForBranch = await octokit.pulls.list({
+    owner: 'electron',
+    repo: 'electron',
+    state: 'open',
+    head: `electron:${targetBranch}`
+  });
+  if (prsForBranch.data.length === 1) {
+    return prsForBranch.data[0].number;
+  } else {
+    return null;
+  }
+}
+
+function useAppVeyorImage (targetBranch, options) {
+  const validJobs = Object.keys(appVeyorJobs);
+  if (options.job) {
+    assert(validJobs.includes(options.job), `Unknown AppVeyor CI job name: ${options.job}.  Valid values are: ${validJobs}.`);
+    callAppVeyorBuildJobs(targetBranch, options.job, options);
+  } else {
+    validJobs.forEach((job) => callAppVeyorBuildJobs(targetBranch, job, options));
+  }
+}
+
+async function callAppVeyorBuildJobs (targetBranch, job, options) {
+  console.log(`Using AppVeyor image ${options.version} for ${job}`);
+
+  const pullRequestId = await getPullRequestId(targetBranch);
+  const environmentVariables = {
+    APPVEYOR_BUILD_WORKER_CLOUD: DEFAULT_BUILD_CLOUD,
+    APPVEYOR_BUILD_WORKER_IMAGE: options.version,
+    ELECTRON_OUT_DIR: 'Default',
+    ELECTRON_ENABLE_STACK_DUMPING: 1,
+    ELECTRON_ALSO_LOG_TO_STDERR: 1,
+    GOMA_FALLBACK_ON_AUTH_FAILURE: true,
+    DEPOT_TOOLS_WIN_TOOLCHAIN: 0,
+    PYTHONIOENCODING: 'UTF-8'
+  };
+
+  const requestOpts = {
+    url: APPVEYOR_JOB_URL,
+    auth: {
+      bearer: process.env.APPVEYOR_TOKEN
+    },
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      accountName: 'electron-bot',
+      projectSlug: appVeyorJobs[job],
+      branch: targetBranch,
+      pullRequestId: pullRequestId || undefined,
+      commitId: options.commit || undefined,
+      environmentVariables
+    }),
+    method: 'POST'
+  };
+
+  try {
+    const { version } = await makeRequest(requestOpts);
+    const buildUrl = `https://ci.appveyor.com/project/electron-bot/${appVeyorJobs[job]}/build/${version}`;
+    console.log(`AppVeyor CI request for ${job} successful.  Check status at ${buildUrl}`);
+  } catch (err) {
+    console.log('Could not call AppVeyor: ', err);
+  }
+}
+
+async function bakeAppVeyorImage (targetBranch, options) {
+  console.log(`Baking a new AppVeyor image for ${options.version}, on build cloud ${options.cloudId}`);
+
+  const environmentVariables = {
+    APPVEYOR_BUILD_WORKER_CLOUD: DEFAULT_BUILD_CLOUD,
+    APPVEYOR_BUILD_WORKER_IMAGE: DEFAULT_BAKE_BASE_IMAGE,
+    APPVEYOR_BAKE_IMAGE: options.version
+  };
+
+  const requestOpts = {
+    url: APPVEYOR_JOB_URL,
+    auth: {
+      bearer: process.env.APPVEYOR_TOKEN
+    },
+    headers: {
+      'Content-Type': 'application/json'
+    },
+    body: JSON.stringify({
+      accountName: 'electron-bot',
+      projectSlug: appveyorBakeJob,
+      branch: targetBranch,
+      commitId: options.commit || undefined,
+      environmentVariables
+    }),
+    method: 'POST'
+  };
+
+  try {
+    const { version } = await makeRequest(requestOpts);
+    const bakeUrl = `https://ci.appveyor.com/project/electron-bot/${appveyorBakeJob}/build/${version}`;
+    console.log(`AppVeyor image bake request for ${options.version} successful.  Check bake status at ${bakeUrl}`);
+  } catch (err) {
+    console.log('Could not call AppVeyor: ', err);
+  }
+}
+
+async function prepareAppVeyorImage (opts) {
+  const branch = await handleGitCall(['rev-parse', '--abbrev-ref', 'HEAD'], ELECTRON_DIR);
+  if (ROLLER_BRANCH_PATTERN.test(branch)) {
+    useAppVeyorImage(branch, { ...opts, version: DEFAULT_BUILD_IMAGE, cloudId: DEFAULT_BUILD_CLOUD_ID });
+  } else {
+    // eslint-disable-next-line no-control-regex
+    const versionRegex = new RegExp('chromium_version\':\n +\'(.+?)\',', 'm');
+    const deps = fs.readFileSync(path.resolve(__dirname, '..', 'DEPS'), 'utf8');
+    const [, CHROMIUM_VERSION] = versionRegex.exec(deps);
+
+    const cloudId = opts.cloudId || DEFAULT_BUILD_CLOUD_ID;
+    const imageVersion = opts.imageVersion || `e-${CHROMIUM_VERSION}`;
+    const image = await checkAppVeyorImage({ cloudId, imageVersion });
+
+    if (image && image.name) {
+      console.log(`Image exists for ${image.name}. Continuing AppVeyor jobs using ${cloudId}.\n`);
+    } else {
+      console.log(`No AppVeyor image found for ${imageVersion} in ${cloudId}.
+                   Creating new image for ${imageVersion}, using Chromium ${CHROMIUM_VERSION} - job will run after image is baked.`);
+      await bakeAppVeyorImage(branch, { ...opts, version: imageVersion, cloudId });
+
+      // write image to temp file if running on CI
+      if (process.env.CI) fs.writeFileSync('./image_version.txt', imageVersion);
+    }
+  }
+}
+
+module.exports = prepareAppVeyorImage;
+
+//   Load or bake AppVeyor images for Windows CI.
+//   Usage: prepare-appveyor.js [--cloudId=CLOUD_ID] [--appveyorJobId=xxx] [--imageVersion=xxx]
+//   [--commit=sha] [--branch=branch_name]
+if (require.main === module) {
+  const args = require('minimist')(process.argv.slice(2));
+  prepareAppVeyorImage(args)
+    .catch((err) => {
+      console.error(err);
+      process.exit(1);
+    });
+}

script/release/ci-release-build.js

@@ -204,7 +204,7 @@ async function callAppVeyor (targetBranch, job, options) {
   console.log(`Triggering AppVeyor to run build job: ${job} on branch: ${targetBranch} with release flag.`);
   const environmentVariables = {
     ELECTRON_RELEASE: 1,
-    APPVEYOR_BUILD_WORKER_CLOUD: 'libcc-20'
+    APPVEYOR_BUILD_WORKER_CLOUD: 'electronhq-16-core'
   };
 
   if (!options.ghRelease) {

script/setup-win-for-dev.bat

@@ -1,9 +1,8 @@
 REM Parameters vs_buildtools.exe download link and wsdk version
 @ECHO OFF
 
-SET buildtools_link=https://download.visualstudio.microsoft.com/download/pr/d7691cc1-82e6-434f-8e9f-a612f85b4b76/c62179f8cbbb58d4af22c21e8d4e122165f21615f529c94fad5cc7e012f1ef08/vs_BuildTools.exe
+SET wsdk10_link=https://go.microsoft.com/fwlink/?linkid=2164145
-SET wsdk10_link=https://go.microsoft.com/fwlink/p/?LinkId=845298
+SET wsdk=10SDK.20348
-SET wsdk=10SDK.18362
 
 REM Check for disk space
 Rem        543210987654321
@@ -44,24 +43,6 @@ IF NOT "%1"=="" (
 
 if not exist "C:\TEMP\" mkdir C:\TEMP
 
-REM Download vs_buildtools.exe to C:\TEMP\vs_buildtools.exe
-powershell -command "& { iwr %buildtools_link% -OutFile C:\TEMP\vs_buildtools.exe }"
-
-REM Install Visual Studio Toolchain
-C:\TEMP\vs_buildtools.exe --quiet --wait --norestart --nocache ^
-    --installPath "%ProgramFiles(x86)%/Microsoft Visual Studio/2019/Community" ^
-    --add Microsoft.VisualStudio.Workload.VCTools ^
-    --add Microsoft.VisualStudio.Component.VC.140 ^
-    --add Microsoft.VisualStudio.Component.VC.ATLMFC ^
-    --add Microsoft.VisualStudio.Component.VC.Tools.ARM64 ^
-    --add Microsoft.VisualStudio.Component.VC.MFC.ARM64 ^
-    --add Microsoft.VisualStudio.Component.Windows%wsdk% ^
-    --includeRecommended
-
-REM Install Windows SDK
-powershell -command "& { iwr %wsdk10_link% -OutFile C:\TEMP\wsdk10.exe }"
-C:\TEMP\wsdk10.exe /features /quiet
-
 REM Install chocolatey to further install dependencies
 set chocolateyUseWindowsCompression='true'
 @"%SystemRoot%\System32\WindowsPowerShell\v1.0\powershell.exe" ^
@@ -69,14 +50,19 @@ set chocolateyUseWindowsCompression='true'
     -Command "iex ((New-Object System.Net.WebClient).DownloadString('https://chocolatey.org/install.ps1'))"
 SET "PATH=%PATH%;%ALLUSERSPROFILE%\chocolatey\bin"
 
-REM Install nodejs python git and yarn needed dependencies
+REM Install Visual Studio Toolchain
-choco install -y nodejs python2 git yarn windows-sdk-10-version-1903-windbg
+choco install visualstudio2019buildtools --package-parameters "--quiet --wait --norestart --nocache  --installPath ""%ProgramFiles(x86)%/Microsoft Visual Studio/2019/Community"" --add Microsoft.VisualStudio.Workload.VCTools --add Microsoft.VisualStudio.Component.VC.140 --add Microsoft.VisualStudio.Component.VC.ATLMFC --add Microsoft.VisualStudio.Component.VC.Tools.ARM64 --add Microsoft.VisualStudio.Component.VC.MFC.ARM64 --add Microsoft.VisualStudio.Component.Windows%wsdk% --includeRecommended"
-call C:\ProgramData\chocolatey\bin\RefreshEnv.cmd
-SET PATH=C:\Python27\;C:\Python27\Scripts;%PATH%
 
-pip install pywin32
+REM Install Windows SDK
+powershell -command "& { iwr %wsdk10_link% -OutFile C:\TEMP\wsdk10.exe }"
+C:\TEMP\wsdk10.exe /features /quiet
+
+REM Install nodejs python git and yarn needed dependencies
+choco install -y nodejs-lts python2 git yarn
+choco install python --version 3.7.9
+choco install windows-sdk-10-version-2004-windbg
 call C:\ProgramData\chocolatey\bin\RefreshEnv.cmd
-pip2 install pywin32
+SET PATH=C:\Python27\;C:\Python27\Scripts;C:\Python39\;C:\Python39\Scripts;%PATH%
 
 REM Setup Depot Tools
 git clone https://chromium.googlesource.com/chromium/tools/depot_tools.git C:\depot_tools

spec/api-browser-view-spec.ts

@@ -63,7 +63,7 @@ describe('BrowserView module', () => {
     });
 
     // Linux and arm64 platforms (WOA and macOS) do not return any capture sources
-    ifit(process.platform !== 'linux' && process.arch !== 'arm64')('sets the background color to transparent if none is set', async () => {
+    ifit(process.platform === 'darwin' && process.arch === 'x64')('sets the background color to transparent if none is set', async () => {
       const display = screen.getPrimaryDisplay();
       const WINDOW_BACKGROUND_COLOR = '#55ccbb';
 
@@ -87,7 +87,7 @@ describe('BrowserView module', () => {
     });
 
     // Linux and arm64 platforms (WOA and macOS) do not return any capture sources
-    ifit(process.platform !== 'linux' && process.arch !== 'arm64')('successfully applies the background color', async () => {
+    ifit(process.platform === 'darwin' && process.arch === 'x64')('successfully applies the background color', async () => {
       const WINDOW_BACKGROUND_COLOR = '#55ccbb';
       const VIEW_BACKGROUND_COLOR = '#ff00ff';
       const display = screen.getPrimaryDisplay();

spec/api-browser-window-spec.ts

@@ -5519,7 +5519,7 @@ describe('BrowserWindow module', () => {
     });
 
     // Linux and arm64 platforms (WOA and macOS) do not return any capture sources
-    ifit(process.platform !== 'linux' && process.arch !== 'arm64')('should not display a visible background', async () => {
+    ifit(process.platform === 'darwin' && process.arch !== 'x64')('should not display a visible background', async () => {
       const display = screen.getPrimaryDisplay();
 
       const backgroundWindow = new BrowserWindow({
@@ -5557,7 +5557,7 @@ describe('BrowserWindow module', () => {
       expect(areColorsSimilar(rightHalfColor, HexColors.RED)).to.be.true();
     });
 
-    ifit(process.platform !== 'linux' && process.arch !== 'arm64')('Allows setting a transparent window via CSS', async () => {
+    ifit(process.platform === 'darwin')('Allows setting a transparent window via CSS', async () => {
       const display = screen.getPrimaryDisplay();
 
       const backgroundWindow = new BrowserWindow({
@@ -5597,7 +5597,8 @@ describe('BrowserWindow module', () => {
   describe('"backgroundColor" option', () => {
     afterEach(closeAllWindows);
 
-    ifit(process.platform !== 'linux' && process.arch !== 'arm64')('should display the set color', async () => {
+    // Linux/WOA doesn't return any capture sources.
+    ifit(process.platform === 'darwin')('should display the set color', async () => {
       const display = screen.getPrimaryDisplay();
 
       const w = new BrowserWindow({