diff --git a/.circleci/config.yml b/.circleci/config.yml
index c7c402dbd76a..2ac7b6319e51 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -119,9 +119,7 @@ build-steps: &build-steps
           if [ "$RUN_TESTS" != "false" ]; then
             cd src
             ninja -C out/Default third_party/electron_node:headers
-            export npm_config_nodedir="$PWD/out/Default/gen/node_headers"
-            (cd electron/spec && npm install)
-            python electron/script/lib/dbus_mock.py ./out/Default/electron electron/spec --ci --enable-logging
+            (cd electron && npm run test -- --ci --enable-logging)
           fi
     - <<: *notify-slack-failure
     - <<: *notify-slack-success
@@ -197,9 +195,7 @@ mac-build-steps: &mac-build-steps
           if [ "$RUN_TESTS" != "false" ]; then
             cd src
             ninja -C out/Default third_party/electron_node:headers
-            export npm_config_nodedir="$PWD/out/Default/gen/node_headers"
-            (cd electron/spec && npm install)
-            ./out/Default/Electron.app/Contents/MacOS/Electron electron/spec --ci --enable-logging
+            (cd electron && npm run test -- --ci --enable-logging)
           fi
     - <<: *notify-slack-failure
     - <<: *notify-slack-success
diff --git a/.gitignore b/.gitignore
index a12eb1a963d0..1b1b040dc595 100644
--- a/.gitignore
+++ b/.gitignore
@@ -56,3 +56,6 @@ compile_commands.json
 # Generated API definitions
 electron-api.json
 electron.d.ts
+
+# Spec hash calculation
+spec/.hash
diff --git a/DEPS b/DEPS
index d678ffd44729..c417b4625d40 100644
--- a/DEPS
+++ b/DEPS
@@ -61,6 +61,15 @@ hooks = [
     'pattern': 'src/electron/package.json',
     'name': 'electron_npm_deps'
   },
+  {
+    'action': [
+      'python',
+      '-c',
+      'import os; os.chdir("src"); os.chdir("electron/spec"); os.system("npm install")',
+    ],
+    'pattern': 'src/electron/spec/package.json',
+    'name': 'electron_spec_npm_deps'
+  },
 ]
 
 recursedeps = [
diff --git a/appveyor.yml b/appveyor.yml
index 48f7652fd8d3..005b85713409 100644
--- a/appveyor.yml
+++ b/appveyor.yml
@@ -34,14 +34,10 @@ test_script:
   - ps: >-
       if ($env:GN_CONFIG -eq 'testing') {
         ninja -C out/Default third_party/electron_node:headers
-        $env:npm_config_nodedir="$pwd/out/Default/gen/node_headers"
-        $env:npm_config_msvs_version="2017"
         New-Item .\out\Default\gen\node_headers\Release -Type directory
         Copy-Item -path .\out\Default\electron.lib -destination .\out\Default\gen\node_headers\Release\node.lib
-        Push-Location; cd electron/spec
-        npm install
-        Pop-Location
       } else {
         echo "Skipping tests for $env:GN_CONFIG build"
       }
-  - if "%GN_CONFIG%"=="testing" ( echo Running test suite & .\out\Default\electron.exe electron\spec --ci )
+  - cd electron
+  - if "%GN_CONFIG%"=="testing" ( echo Running test suite & npm run test -- --ci )
diff --git a/package.json b/package.json
index 955dce9d3ebc..4fd03a94e3b3 100644
--- a/package.json
+++ b/package.json
@@ -14,6 +14,7 @@
     "electabul": "~0.0.4",
     "electron-docs-linter": "^2.3.4",
     "electron-typescript-definitions": "^2.0.0",
+    "folder-hash": "^2.1.1",
     "github": "^9.2.0",
     "html-entities": "^1.2.1",
     "husky": "^0.14.3",
@@ -72,7 +73,7 @@
     "release": "node ./script/release.js",
     "repl": "python ./script/start.py --interactive",
     "start": "python ./script/start.py",
-    "test": "python ./script/test.py"
+    "test": "node ./script/spec-runner.js electron/spec"
   },
   "license": "MIT",
   "author": "Electron Community",
diff --git a/script/spec-runner.js b/script/spec-runner.js
new file mode 100644
index 000000000000..3e26838092ce
--- /dev/null
+++ b/script/spec-runner.js
@@ -0,0 +1,82 @@
+const cp = require('child_process')
+const crypto = require('crypto')
+const fs = require('fs')
+const { hashElement } = require('folder-hash')
+const path = require('path')
+
+const BASE = path.resolve(__dirname, '../..')
+const NPM_CMD = process.platform === 'win32' ? 'npm.cmd' : 'npm'
+const OUT_DIR = process.env.ELECTRON_SPEC_OUT_DIR || 'Default'
+
+const specHashPath = path.resolve(__dirname, '../spec/.hash')
+
+const [lastSpecHash, lastSpecInstallHash] = fs.existsSync(specHashPath)
+  ? fs.readFileSync(specHashPath, 'utf8').split('\n')
+  : ['invalid', 'invalid']
+
+getSpecHash().then(([currentSpecHash, currentSpecInstallHash]) => {
+  const specChanged = currentSpecHash !== lastSpecHash
+  const installChanged = lastSpecInstallHash !== currentSpecInstallHash
+  if (specChanged || installChanged) {
+    const out = cp.spawnSync(NPM_CMD, ['install'], {
+      env: Object.assign({}, process.env, {
+        npm_config_nodedir: path.resolve(BASE, `out/${OUT_DIR}/gen/node_headers`),
+        npm_config_msvs_version: '2017'
+      }),
+      cwd: path.resolve(__dirname, '../spec')
+    })
+    if (out.status !== 0) {
+      console.error('Failed to npm install in the spec folder')
+      process.exit(1)
+    }
+    return getSpecHash()
+      .then(([newSpecHash, newSpecInstallHash]) => {
+        fs.writeFileSync(specHashPath, `${newSpecHash}\n${newSpecInstallHash}`)
+      })
+  }
+}).then(() => {
+  let exe = path.resolve(BASE, getElectronExec())
+  const args = process.argv.slice(2)
+  if (process.platform === 'linux') {
+    args.unshift(path.resolve(__dirname, 'lib/dbus_mock.py'), exe)
+    exe = 'python'
+  }
+  const child = cp.spawn(exe, args, {
+    cwd: path.resolve(__dirname, '../..'),
+    stdio: 'inherit'
+  })
+  child.on('exit', (code) => {
+    process.exit(code)
+  })
+})
+
+function getElectronExec () {
+  switch (process.platform) {
+    case 'darwin':
+      return 'out/Default/Electron.app/Contents/MacOS/Electron'
+    case 'win32':
+      return 'out/Default/electron.exe'
+    case 'linux':
+      return 'out/Default/electron'
+    default:
+      throw new Error('Unknown path')
+  }
+}
+
+function getSpecHash () {
+  return Promise.all([
+    new Promise((resolve) => {
+      const hasher = crypto.createHash('SHA256')
+      hasher.update(fs.readFileSync(path.resolve(__dirname, '../spec/package.json')))
+      hasher.update(fs.readFileSync(path.resolve(__dirname, '../spec/package-lock.json')))
+      resolve(hasher.digest('hex'))
+    }),
+    new Promise((resolve, reject) => {
+      const specNodeModulesPath = path.resolve(__dirname, '../spec/node_modules')
+      if (!fs.existsSync(specNodeModulesPath)) {
+        return resolve('invalid')
+      }
+      hashElement(specNodeModulesPath).then((result) => resolve(result.hash)).catch(reject)
+    })
+  ])
+}