Speed up lint-deps

ts/util/lint/linter.ts

@@ -1,26 +1,27 @@
 // tslint:disable no-console
-
+import * as fs from 'fs';
-import { readFileSync } from 'fs';
 import { join, relative } from 'path';
 import normalizePath from 'normalize-path';
-
+import pMap from 'p-map';
-import { sync as fgSync } from 'fast-glob';
+import FastGlob from 'fast-glob';
-import { forEach, some, values } from 'lodash';
 
 import { ExceptionType, REASONS, RuleType } from './types';
 import { ENCODING, loadJSON, sortExceptions } from './util';
 
 const ALL_REASONS = REASONS.join('|');
-const now = new Date();
 
-function getExceptionKey(exception: any) {
+function getExceptionKey(
+  exception: Pick<ExceptionType, 'rule' | 'path' | 'lineNumber'>
+): string {
   return `${exception.rule}-${exception.path}-${exception.lineNumber}`;
 }
 
-function createLookup(list: Array<any>) {
+function createLookup(
+  list: ReadonlyArray<ExceptionType>
+): { [key: string]: ExceptionType } {
   const lookup = Object.create(null);
 
-  forEach(list, exception => {
+  list.forEach((exception: ExceptionType) => {
     const key = getExceptionKey(exception);
 
     if (lookup[key]) {
@@ -39,16 +40,7 @@ const basePath = join(__dirname, '../../..');
 
 const searchPattern = normalizePath(join(basePath, '**/*.{js,ts,tsx}'));
 
-const rules: Array<RuleType> = loadJSON(rulesPath);
+const excludedFilesRegexps = [
-const exceptions: Array<ExceptionType> = loadJSON(exceptionsPath);
-const exceptionsLookup = createLookup(exceptions);
-let scannedCount = 0;
-
-const allSourceFiles = fgSync(searchPattern, { onlyFiles: true });
-
-const results: Array<ExceptionType> = [];
-
-const excludedFiles = [
   // Non-distributed files
   '\\.d\\.ts$',
 
@@ -295,10 +287,10 @@ const excludedFiles = [
   '^node_modules/webpack-hot-middleware/.+',
   '^node_modules/webpack-merge/.+',
   '^node_modules/webpack/.+',
-];
+].map(str => new RegExp(str));
 
 function setupRules(allRules: Array<RuleType>) {
-  forEach(allRules, (rule, index) => {
+  allRules.forEach((rule: RuleType, index: number) => {
     if (!rule.name) {
       throw new Error(`Rule at index ${index} is missing a name`);
     }
@@ -311,93 +303,108 @@ function setupRules(allRules: Array<RuleType>) {
   });
 }
 
-setupRules(rules);
+async function main(): Promise<void> {
+  const now = new Date();
 
-forEach(allSourceFiles, file => {
+  const rules: Array<RuleType> = loadJSON(rulesPath);
-  const relativePath = relative(basePath, file).replace(/\\/g, '/');
+  setupRules(rules);
-  if (
-    some(excludedFiles, excluded => {
-      const regex = new RegExp(excluded);
 
-      return regex.test(relativePath);
+  const exceptions: Array<ExceptionType> = loadJSON(exceptionsPath);
-    })
+  const exceptionsLookup = createLookup(exceptions);
-  ) {
+
-    return;
+  const results: Array<ExceptionType> = [];
+  let scannedCount = 0;
+
+  await pMap(
+    await FastGlob(searchPattern, { onlyFiles: true }),
+    async (file: string) => {
+      const relativePath = relative(basePath, file).replace(/\\/g, '/');
+
+      const isFileExcluded = excludedFilesRegexps.some(excludedRegexp =>
+        excludedRegexp.test(relativePath)
+      );
+      if (isFileExcluded) {
+        return;
+      }
+
+      scannedCount += 1;
+
+      const lines = (await fs.promises.readFile(file, ENCODING)).split(/\r?\n/);
+
+      rules.forEach((rule: RuleType) => {
+        const excludedModules = rule.excludedModules || [];
+        if (excludedModules.some(module => relativePath.startsWith(module))) {
+          return;
+        }
+
+        lines.forEach((line: string, lineIndex: number) => {
+          if (!rule.regex.test(line)) {
+            return;
+          }
+          // recreate this rule since it has g flag, and carries local state
+          if (rule.expression) {
+            rule.regex = new RegExp(rule.expression, 'g');
+          }
+
+          const lineNumber = lineIndex + 1;
+
+          const exceptionKey = getExceptionKey({
+            rule: rule.name,
+            path: relativePath,
+            lineNumber,
+          });
+
+          const exception = exceptionsLookup[exceptionKey];
+          if (exception && (!exception.line || exception.line === line)) {
+            // tslint:disable-next-line no-dynamic-delete
+            delete exceptionsLookup[exceptionKey];
+
+            return;
+          }
+
+          results.push({
+            rule: rule.name,
+            path: relativePath,
+            line: line.length < 300 ? line : undefined,
+            lineNumber,
+            reasonCategory: ALL_REASONS,
+            updated: now.toJSON(),
+            reasonDetail: '<optional>',
+          });
+        });
+      });
+    },
+    // Without this, we may run into "too many open files" errors.
+    { concurrency: 100 }
+  );
+
+  const unusedExceptions = Object.values(exceptionsLookup);
+
+  console.log(
+    `${scannedCount} files scanned.`,
+    `${results.length} questionable lines,`,
+    `${unusedExceptions.length} unused exceptions,`,
+    `${exceptions.length} total exceptions.`
+  );
+
+  if (results.length === 0 && unusedExceptions.length === 0) {
+    process.exit();
   }
 
-  scannedCount += 1;
-
-  const fileContents = readFileSync(file, ENCODING);
-  const lines = fileContents.split('\n');
-
-  forEach(rules, (rule: RuleType) => {
-    const excludedModules = rule.excludedModules || [];
-    if (some(excludedModules, module => relativePath.startsWith(module))) {
-      return;
-    }
-
-    forEach(lines, (rawLine, lineIndex) => {
-      const line = rawLine.replace(/\r/g, '');
-
-      if (!rule.regex.test(line)) {
-        return;
-      }
-      // recreate this rule since it has g flag, and carries local state
-      if (rule.expression) {
-        rule.regex = new RegExp(rule.expression, 'g');
-      }
-
-      const path = relativePath;
-      const lineNumber = lineIndex + 1;
-
-      const exceptionKey = getExceptionKey({
-        rule: rule.name,
-        path: relativePath,
-        lineNumber,
-      });
-
-      const exception = exceptionsLookup[exceptionKey];
-      if (exception && (!exception.line || exception.line === line)) {
-        // tslint:disable-next-line no-dynamic-delete
-        delete exceptionsLookup[exceptionKey];
-
-        return;
-      }
-
-      results.push({
-        rule: rule.name,
-        path,
-        line: line.length < 300 ? line : undefined,
-        lineNumber,
-        reasonCategory: ALL_REASONS,
-        updated: now.toJSON(),
-        reasonDetail: '<optional>',
-      });
-    });
-  });
-});
-
-const unusedExceptions = values(exceptionsLookup);
-
-console.log(
-  `${scannedCount} files scanned.`,
-  `${results.length} questionable lines,`,
-  `${unusedExceptions.length} unused exceptions,`,
-  `${exceptions.length} total exceptions.`
-);
-
-if (results.length === 0 && unusedExceptions.length === 0) {
-  process.exit();
-}
-
-console.log();
-console.log('Questionable lines:');
-console.log(JSON.stringify(sortExceptions(results), null, '  '));
-
-if (unusedExceptions.length) {
   console.log();
-  console.log('Unused exceptions!');
+  console.log('Questionable lines:');
-  console.log(JSON.stringify(sortExceptions(unusedExceptions), null, '  '));
+  console.log(JSON.stringify(sortExceptions(results), null, '  '));
+
+  if (unusedExceptions.length) {
+    console.log();
+    console.log('Unused exceptions!');
+    console.log(JSON.stringify(sortExceptions(unusedExceptions), null, '  '));
+  }
+
+  process.exit(1);
 }
 
-process.exit(1);
+main().catch(err => {
+  console.error(err);
+  process.exit(1);
+});