Fix windows fast-glob usage in lint-deps test task

* Introduce normalize-path wherever we use fastGlob

* CI: Update yarn version; install yarn via npm on windows

* Add more logging to Grunt file spectron usage

* Lock core.js to what it was resolving to before: 2.4.1

* test/index.html: Remove nonexistent test file

* test/index.html: Remove missing registration.js

* preload.js: Introduce client-side logging for load failures

* Gruntfile: Introduce better debuggability if prod test fails

* Reintroduce glob for searches inside asar

.travis.yml

@@ -5,7 +5,7 @@ os:
   - linux
 dist: trusty
 before_install:
-  - npm install -g yarn@1.17.3
+  - npm install -g yarn@1.22.0
 install:
   - yarn install --frozen-lockfile
 script:

Gruntfile.js

@@ -1,4 +1,4 @@
-const path = require('path');
+const { join } = require('path');
 const packageJson = require('./package.json');
 const importOnce = require('node-sass-import-once');
 const rimraf = require('rimraf');
@@ -223,9 +223,13 @@ module.exports = grunt => {
     const { Application } = spectron;
     const electronBinary =
       process.platform === 'win32' ? 'electron.cmd' : 'electron';
+
+    const path = join(__dirname, 'node_modules', '.bin', electronBinary);
+    const args = [join(__dirname, 'main.js')];
+    console.log('Starting path', path, 'with args', args);
     const app = new Application({
-      path: path.join(__dirname, 'node_modules', '.bin', electronBinary),
+      path,
-      args: [path.join(__dirname, 'main.js')],
+      args,
       env: {
         NODE_ENV: environment,
       },
@@ -239,19 +243,24 @@ module.exports = grunt => {
 
     app
       .start()
-      .then(() =>
+      .then(() => {
-        app.client.waitUntil(
+        console.log('App started. Now waiting for test results...');
+        return app.client.waitUntil(
           () =>
             app.client
               .execute(getMochaResults)
               .then(data => Boolean(data.value)),
           25000,
           'Expected to find window.mochaResults set!'
-        )
+        );
-      )
+      })
       .then(() => app.client.execute(getMochaResults))
       .then(data => {
         const results = data.value;
+        if (!results) {
+          failure = () => grunt.fail.fatal("Couldn't extract test results.");
+          return app.client.log('browser');
+        }
         if (results.failures > 0) {
           console.error(results.reports);
           failure = () =>
@@ -368,14 +377,23 @@ module.exports = grunt => {
       // A simple test to verify a visible window is opened with a title
       const { Application } = spectron;
 
+      const path = [dir, config.exe].join('/');
+      console.log('Starting path', path);
       const app = new Application({
-        path: [dir, config.exe].join('/'),
+        path,
-        requireName: 'unused',
       });
 
-      app
+      const sleep = millis =>
-        .start()
+        new Promise(resolve => setTimeout(resolve, millis));
-        .then(() => app.client.getWindowCount())
+
+      Promise.race([app.start(), sleep(15000)])
+        .then(() => {
+          if (!app.isRunning()) {
+            throw new Error('Application failed to start');
+          }
+
+          return app.client.getWindowCount();
+        })
         .then(count => {
           assert.equal(count, 1);
           console.log('window opened');
@@ -405,6 +423,17 @@ module.exports = grunt => {
               grunt.fail.fatal(`Test failed: ${error.message} ${error.stack}`);
             })
         )
+        .catch(error => {
+          console.log('Main process logs:');
+          app.client.getMainProcessLogs().then(logs => {
+            logs.forEach(log => {
+              console.log(log);
+            });
+
+            // Test failed!
+            grunt.fail.fatal(`Failure! ${error.message} ${error.stack}`);
+          });
+        })
         .then(done);
     }
   );

app/attachments.js

@@ -3,9 +3,12 @@ const path = require('path');
 const { app, dialog, shell, remote } = require('electron');
 
 const fastGlob = require('fast-glob');
+const glob = require('glob');
+const pify = require('pify');
 const fse = require('fs-extra');
 const toArrayBuffer = require('to-arraybuffer');
 const { map, isArrayBuffer, isString } = require('lodash');
+const normalizePath = require('normalize-path');
 const sanitizeFilename = require('sanitize-filename');
 const getGuid = require('uuid/v4');
 
@@ -25,7 +28,7 @@ const DRAFT_PATH = 'drafts.noindex';
 
 exports.getAllAttachments = async userDataPath => {
   const dir = exports.getPath(userDataPath);
-  const pattern = path.join(dir, '**', '*');
+  const pattern = normalizePath(path.join(dir, '**', '*'));
 
   const files = await fastGlob(pattern, { onlyFiles: true });
   return map(files, file => path.relative(dir, file));
@@ -33,7 +36,7 @@ exports.getAllAttachments = async userDataPath => {
 
 exports.getAllStickers = async userDataPath => {
   const dir = exports.getStickersPath(userDataPath);
-  const pattern = path.join(dir, '**', '*');
+  const pattern = normalizePath(path.join(dir, '**', '*'));
 
   const files = await fastGlob(pattern, { onlyFiles: true });
   return map(files, file => path.relative(dir, file));
@@ -41,7 +44,7 @@ exports.getAllStickers = async userDataPath => {
 
 exports.getAllDraftAttachments = async userDataPath => {
   const dir = exports.getDraftPath(userDataPath);
-  const pattern = path.join(dir, '**', '*');
+  const pattern = normalizePath(path.join(dir, '**', '*'));
 
   const files = await fastGlob(pattern, { onlyFiles: true });
   return map(files, file => path.relative(dir, file));
@@ -51,7 +54,9 @@ exports.getBuiltInImages = async () => {
   const dir = path.join(__dirname, '../images');
   const pattern = path.join(dir, '**', '*.svg');
 
-  const files = await fastGlob(pattern, { onlyFiles: true });
+  // Note: we cannot use fast-glob here because, inside of .asar files, readdir will not
+  //   honor the withFileTypes flag: https://github.com/electron/electron/issues/19074
+  const files = await pify(glob)(pattern, { nodir: true });
   return map(files, file => path.relative(dir, file));
 };
 

appveyor.yml

@@ -9,7 +9,7 @@ install:
   - systeminfo | findstr /C:"OS"
   - set PATH=C:\Ruby23-x64\bin;%PATH%
   - ps: Install-Product node 12.13.0 x64
-  - ps: .\build\install-yarn.ps1
+  - npm install -g yarn@1.22.0
   - yarn install --frozen-lockfile
 
 build_script:

build/install-yarn.ps1

@@ -1,12 +0,0 @@
-# via https://gist.github.com/FeodorFitsner/1056703ec92bd6a3012c15fe78a5e162
-
-Write-Host "Installing Yarn..." -ForegroundColor Cyan
-
-Write-Host "Downloading..."
-$msiPath = "$env:TEMP\yarn.msi"
-(New-Object Net.WebClient).DownloadFile('https://github.com/yarnpkg/yarn/releases/download/v1.17.3/yarn-1.17.3.msi', $msiPath)
-
-Write-Host "Installing..."
-cmd /c start /wait msiexec /i "$msiPath" /quiet
-
-Write-Host "Yarn installed" -ForegroundColor Green

package.json

@@ -88,6 +88,7 @@
     "form-data": "2.3.2",
     "fs-extra": "5.0.0",
     "fuse.js": "3.4.4",
+    "glob": "7.1.6",
     "google-libphonenumber": "3.2.6",
     "got": "8.2.0",
     "he": "1.2.0",
@@ -173,6 +174,7 @@
     "@types/memoizee": "0.4.2",
     "@types/mkdirp": "0.5.2",
     "@types/mocha": "5.0.0",
+    "@types/normalize-path": "3.0.0",
     "@types/pify": "3.0.2",
     "@types/qs": "6.5.1",
     "@types/react": "16.8.5",
@@ -200,7 +202,7 @@
     "babel-plugin-lodash": "3.3.4",
     "bower": "1.8.2",
     "chai": "4.1.2",
-    "core-js": "2.4.0",
+    "core-js": "2.4.1",
     "cross-env": "5.2.0",
     "css-loader": "3.2.0",
     "dashdash": "1.14.1",

preload.js

@@ -1,5 +1,8 @@
 /* global Whisper, window */
 
+/* eslint-disable global-require, no-inner-declarations */
+
+try {
   const electron = require('electron');
   const semver = require('semver');
   const curve = require('curve25519-n');
@@ -356,7 +359,8 @@ const externalCurveAsync = {
     createKeyPair: wrapWithPromise(externalCurve.createKeyPair),
     calculateAgreement: wrapWithPromise(externalCurve.calculateAgreement),
     verifySignature: async (...args) => {
-    // The async verifySignature function has a different signature than the sync function
+      // The async verifySignature function has a different signature than the
+      //   sync function
       const verifyFailed = externalCurve.verifySignature(...args);
       if (verifyFailed) {
         throw new Error('Invalid signature');
@@ -398,6 +402,7 @@ if (config.environment === 'test') {
     /* eslint-disable global-require, import/no-extraneous-dependencies */
     window.test = {
       fastGlob: require('fast-glob'),
+      normalizePath: require('normalize-path'),
       fse: require('fs-extra'),
       tmp: require('tmp'),
       path: require('path'),
@@ -406,3 +411,9 @@ if (config.environment === 'test') {
     };
     /* eslint-enable global-require, import/no-extraneous-dependencies */
   }
+} catch (error) {
+  window.log.info('preload error!', error.stack);
+  throw error;
+}
+
+window.log.info('preload complete');

test/backup_test.js

@@ -239,14 +239,23 @@ describe('Backup', () => {
     it('exports then imports to produce the same data we started with', async function thisNeeded() {
       this.timeout(6000);
 
-      const { attachmentsPath, fse, fastGlob, path, tmp } = window.test;
+      const {
+        attachmentsPath,
+        fse,
+        fastGlob,
+        normalizePath,
+        path,
+        tmp,
+      } = window.test;
       const {
         upgradeMessageSchema,
         loadAttachmentData,
       } = window.Signal.Migrations;
 
       const staticKeyPair = await libsignal.KeyHelper.generateIdentityKeyPair();
-      const attachmentsPattern = path.join(attachmentsPath, '**');
+      const attachmentsPattern = normalizePath(
+        path.join(attachmentsPath, '**')
+      );
 
       const OUR_NUMBER = '+12025550000';
       const CONTACT_ONE_NUMBER = '+12025550001';
@@ -528,7 +537,9 @@ describe('Backup', () => {
         console.log(
           'Backup test: Ensure that all attachments made it to backup dir'
         );
-        const backupAttachmentPattern = path.join(backupDir, 'attachments/*');
+        const backupAttachmentPattern = normalizePath(
+          path.join(backupDir, 'attachments/*')
+        );
         const backupAttachments = fastGlob.sync(backupAttachmentPattern);
         console.log({ backupAttachments });
         assert.strictEqual(ATTACHMENT_COUNT, backupAttachments.length);

test/index.html

@@ -433,7 +433,6 @@
   <script type="text/javascript" src="../js/reliable_trigger.js" data-cover></script>
   <script type="text/javascript" src="test.js"></script>
 
-  <script type='text/javascript' src='../js/registration.js' data-cover></script>
   <script type="text/javascript" src="../js/chromium.js" data-cover></script>
   <script type="text/javascript" src="../js/database.js" data-cover></script>
   <script type="text/javascript" src="../js/storage.js" data-cover></script>
@@ -477,7 +476,6 @@
   <script type="text/javascript" src="models/messages_test.js"></script>
 
   <script type="text/javascript" src="libphonenumber_util_test.js"></script>
-  <script type="text/javascript" src="conversation_controller_test.js"></script>
   <script type="text/javascript" src="storage_test.js"></script>
   <script type="text/javascript" src="keychange_listener_test.js"></script>
   <script type="text/javascript" src="reliable_trigger_test.js"></script>

ts/util/lint/linter.ts

@@ -2,6 +2,7 @@
 
 import { readFileSync } from 'fs';
 import { join, relative } from 'path';
+import normalizePath from 'normalize-path';
 
 import { sync as fgSync } from 'fast-glob';
 import { forEach, some, values } from 'lodash';
@@ -36,7 +37,7 @@ const rulesPath = join(__dirname, 'rules.json');
 const exceptionsPath = join(__dirname, 'exceptions.json');
 const basePath = join(__dirname, '../../..');
 
-const searchPattern = join(basePath, '**/*.{js,ts,tsx}');
+const searchPattern = normalizePath(join(basePath, '**/*.{js,ts,tsx}'));
 
 const rules: Array<RuleType> = loadJSON(rulesPath);
 const exceptions: Array<ExceptionType> = loadJSON(exceptionsPath);

yarn.lock

@@ -2081,6 +2081,11 @@
   resolved "https://registry.yarnpkg.com/@types/node/-/node-8.10.40.tgz#4314888d5cd537945d73e9ce165c04cc550144a4"
   integrity sha512-RRSjdwz63kS4u7edIwJUn8NqKLLQ6LyqF/X4+4jp38MBT3Vwetewi2N4dgJEshLbDwNgOJXNYoOwzVZUSSLhkQ==
 
+"@types/normalize-path@3.0.0":
+  version "3.0.0"
+  resolved "https://registry.yarnpkg.com/@types/normalize-path/-/normalize-path-3.0.0.tgz#bb5c46cab77b93350b4cf8d7ff1153f47189ae31"
+  integrity sha512-Nd8y/5t/7CRakPYiyPzr/IAfYusy1FkcZYFEAcoMZkwpJv2n4Wm+olW+e7xBdHEXhOnWdG9ddbar0gqZWS4x5Q==
+
 "@types/pify@3.0.2":
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/@types/pify/-/pify-3.0.2.tgz#1bc75dac43e31dba981c37e0a08edddc1b49cd39"
@@ -5139,19 +5144,14 @@ core-js-pure@^3.0.1:
   resolved "https://registry.yarnpkg.com/core-js-pure/-/core-js-pure-3.2.1.tgz#879a23699cff46175bfd2d09158b5c50645a3c45"
   integrity sha512-+qpvnYrsi/JDeQTArB7NnNc2VoMYLE1YSkziCDHgjexC2KH7OFiGhLUd3urxfyWmNjSwSW7NYXPWHMhuIJx9Ow==
 
-core-js@2.4.0:
+core-js@2.4.1, core-js@^2.4.0:
-  version "2.4.0"
+  version "2.4.1"
-  resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.4.0.tgz#df408ab46d01aff91c01c3e7971935d422c54f81"
+  resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.4.1.tgz#4de911e667b0eae9124e34254b53aea6fc618d3e"
-  integrity sha1-30CKtG0Br/kcAcPnlxk11CLFT4E=
 
 core-js@^1.0.0:
   version "1.2.7"
   resolved "https://registry.yarnpkg.com/core-js/-/core-js-1.2.7.tgz#652294c14651db28fa93bd2d5ff2983a4f08c636"
 
-core-js@^2.4.0:
-  version "2.4.1"
-  resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.4.1.tgz#4de911e667b0eae9124e34254b53aea6fc618d3e"
-
 core-js@^2.4.1:
   version "2.5.4"
   resolved "https://registry.yarnpkg.com/core-js/-/core-js-2.5.4.tgz#f2c8bf181f2a80b92f360121429ce63a2f0aeae0"
@@ -7919,6 +7919,18 @@ glob@7.1.3:
     once "^1.3.0"
     path-is-absolute "^1.0.0"
 
+glob@7.1.6:
+  version "7.1.6"
+  resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.6.tgz#141f33b81a7c2492e125594307480c46679278a6"
+  integrity sha512-LwaxwyZ72Lk7vZINtNNrywX0ZuLyStrdDtabefZKAY5ZGJhVtgdznluResxNmPitE0SAO+O26sWTHeKSI2wMBA==
+  dependencies:
+    fs.realpath "^1.0.0"
+    inflight "^1.0.4"
+    inherits "2"
+    minimatch "^3.0.4"
+    once "^1.3.0"
+    path-is-absolute "^1.0.0"
+
 glob@^6.0.1, glob@^6.0.4:
   version "6.0.4"
   resolved "https://registry.yarnpkg.com/glob/-/glob-6.0.4.tgz#0f08860f6a155127b2fadd4f9ce24b1aab6e4d22"