Speed up lint-deps

ts/util/lint/linter.ts

@@ -1,26 +1,27 @@
 // tslint:disable no-console
-
+import * as fs from 'fs';
-import { readFileSync } from 'fs';
 import { join, relative } from 'path';
 import normalizePath from 'normalize-path';
-
+import pMap from 'p-map';
-import { sync as fgSync } from 'fast-glob';
+import FastGlob from 'fast-glob';
-import { forEach, some, values } from 'lodash';
 
 import { ExceptionType, REASONS, RuleType } from './types';
 import { ENCODING, loadJSON, sortExceptions } from './util';
 
 const ALL_REASONS = REASONS.join('|');
-const now = new Date();
 
-function getExceptionKey(exception: any) {
+function getExceptionKey(
+  exception: Pick<ExceptionType, 'rule' | 'path' | 'lineNumber'>
+): string {
   return `${exception.rule}-${exception.path}-${exception.lineNumber}`;
 }
 
-function createLookup(list: Array<any>) {
+function createLookup(
+  list: ReadonlyArray<ExceptionType>
+): { [key: string]: ExceptionType } {
   const lookup = Object.create(null);
 
-  forEach(list, exception => {
+  list.forEach((exception: ExceptionType) => {
     const key = getExceptionKey(exception);
 
     if (lookup[key]) {
@@ -39,16 +40,7 @@ const basePath = join(__dirname, '../../..');
 
 const searchPattern = normalizePath(join(basePath, '**/*.{js,ts,tsx}'));
 
-const rules: Array<RuleType> = loadJSON(rulesPath);
+const excludedFilesRegexps = [
-const exceptions: Array<ExceptionType> = loadJSON(exceptionsPath);
-const exceptionsLookup = createLookup(exceptions);
-let scannedCount = 0;
-
-const allSourceFiles = fgSync(searchPattern, { onlyFiles: true });
-
-const results: Array<ExceptionType> = [];
-
-const excludedFiles = [
   // Non-distributed files
   '\\.d\\.ts$',
 
@@ -295,10 +287,10 @@ const excludedFiles = [
   '^node_modules/webpack-hot-middleware/.+',
   '^node_modules/webpack-merge/.+',
   '^node_modules/webpack/.+',
-];
+].map(str => new RegExp(str));
 
 function setupRules(allRules: Array<RuleType>) {
-  forEach(allRules, (rule, index) => {
+  allRules.forEach((rule: RuleType, index: number) => {
     if (!rule.name) {
       throw new Error(`Rule at index ${index} is missing a name`);
     }
@@ -311,34 +303,41 @@ function setupRules(allRules: Array<RuleType>) {
   });
 }
 
+async function main(): Promise<void> {
+  const now = new Date();
+
+  const rules: Array<RuleType> = loadJSON(rulesPath);
   setupRules(rules);
 
-forEach(allSourceFiles, file => {
+  const exceptions: Array<ExceptionType> = loadJSON(exceptionsPath);
-  const relativePath = relative(basePath, file).replace(/\\/g, '/');
+  const exceptionsLookup = createLookup(exceptions);
-  if (
-    some(excludedFiles, excluded => {
-      const regex = new RegExp(excluded);
 
-      return regex.test(relativePath);
+  const results: Array<ExceptionType> = [];
-    })
+  let scannedCount = 0;
-  ) {
+
+  await pMap(
+    await FastGlob(searchPattern, { onlyFiles: true }),
+    async (file: string) => {
+      const relativePath = relative(basePath, file).replace(/\\/g, '/');
+
+      const isFileExcluded = excludedFilesRegexps.some(excludedRegexp =>
+        excludedRegexp.test(relativePath)
+      );
+      if (isFileExcluded) {
         return;
       }
 
       scannedCount += 1;
 
-  const fileContents = readFileSync(file, ENCODING);
+      const lines = (await fs.promises.readFile(file, ENCODING)).split(/\r?\n/);
-  const lines = fileContents.split('\n');
 
-  forEach(rules, (rule: RuleType) => {
+      rules.forEach((rule: RuleType) => {
         const excludedModules = rule.excludedModules || [];
-    if (some(excludedModules, module => relativePath.startsWith(module))) {
+        if (excludedModules.some(module => relativePath.startsWith(module))) {
           return;
         }
 
-    forEach(lines, (rawLine, lineIndex) => {
+        lines.forEach((line: string, lineIndex: number) => {
-      const line = rawLine.replace(/\r/g, '');
-
           if (!rule.regex.test(line)) {
             return;
           }
@@ -347,7 +346,6 @@ forEach(allSourceFiles, file => {
             rule.regex = new RegExp(rule.expression, 'g');
           }
 
-      const path = relativePath;
           const lineNumber = lineIndex + 1;
 
           const exceptionKey = getExceptionKey({
@@ -366,7 +364,7 @@ forEach(allSourceFiles, file => {
 
           results.push({
             rule: rule.name,
-        path,
+            path: relativePath,
             line: line.length < 300 ? line : undefined,
             lineNumber,
             reasonCategory: ALL_REASONS,
@@ -375,9 +373,12 @@ forEach(allSourceFiles, file => {
           });
         });
       });
-});
+    },
+    // Without this, we may run into "too many open files" errors.
+    { concurrency: 100 }
+  );
 
-const unusedExceptions = values(exceptionsLookup);
+  const unusedExceptions = Object.values(exceptionsLookup);
 
   console.log(
     `${scannedCount} files scanned.`,
@@ -401,3 +402,9 @@ if (unusedExceptions.length) {
   }
 
   process.exit(1);
+}
+
+main().catch(err => {
+  console.error(err);
+  process.exit(1);
+});