fix: support recursive readdir in Asar files (#41582)

lib/node/asar-fs-wrapper.ts

@@ -1,5 +1,5 @@
 import { Buffer } from 'buffer';
-import { constants } from 'fs';
+import { Dirent, constants } from 'fs';
 import * as path from 'path';
 import * as util from 'util';
 
@@ -46,7 +46,17 @@ process._getOrCreateArchive = getOrCreateArchive;
 
 const asarRe = /\.asar/i;
 
-const { getValidatedPath } = __non_webpack_require__('internal/fs/utils');
+const {
+  getValidatedPath,
+  getOptions,
+  getDirent
+} = __non_webpack_require__('internal/fs/utils');
+
+const {
+  validateBoolean,
+  validateFunction
+} = __non_webpack_require__('internal/validators');
+
 // In the renderer node internals use the node global URL but we do not set that to be
 // the global URL instance.  We need to do instanceof checks against the internal URL impl
 const { URL: NodeURL } = __non_webpack_require__('internal/url');
@@ -78,6 +88,22 @@ const gid = process.getgid?.() ?? 0;
 
 const fakeTime = new Date();
 
+function getDirents (p: string, { 0: names, 1: types }: any[][]): Dirent[] {
+  const info = splitPath(p);
+  const len = names.length;
+  for (let i = 0; i < len; i++) {
+    if (info.isAsar) {
+      const archive = getOrCreateArchive(info.asarPath);
+      const stats = archive!.stat(p);
+      if (!stats) continue;
+      names[i] = getDirent(p, names[i], stats.type);
+    } else {
+      names[i] = getDirent(p, names[i], types[i]);
+    }
+  }
+  return names;
+}
+
 enum AsarFileType {
   kFile = (constants as any).UV_DIRENT_FILE,
   kDirectory = (constants as any).UV_DIRENT_DIR,
@@ -662,13 +688,27 @@ export const wrapFsWithAsar = (fs: Record<string, any>) => {
     return (encoding) ? buffer.toString(encoding) : buffer;
   };
 
+  type ReaddirOptions = { encoding: BufferEncoding | null; withFileTypes?: false, recursive?: false } | undefined | null;
+  type ReaddirCallback = (err: NodeJS.ErrnoException | null, files: string[]) => void;
+
   const { readdir } = fs;
-  fs.readdir = function (pathArgument: string, options?: { encoding?: string | null; withFileTypes?: boolean } | null, callback?: Function) {
+  fs.readdir = function (pathArgument: string, options: ReaddirOptions, callback: ReaddirCallback) {
-    const pathInfo = splitPath(pathArgument);
+    callback = typeof options === 'function' ? options : callback;
-    if (typeof options === 'function') {
+    validateFunction(callback, 'callback');
-      callback = options;
+
-      options = undefined;
+    options = getOptions(options);
+    pathArgument = getValidatedPath(pathArgument);
+
+    if (options?.recursive != null) {
+      validateBoolean(options?.recursive, 'options.recursive');
     }
+
+    if (options?.recursive) {
+      nextTick(callback!, [null, readdirSyncRecursive(pathArgument, options)]);
+      return;
+    }
+
+    const pathInfo = splitPath(pathArgument);
     if (!pathInfo.isAsar) return readdir.apply(this, arguments);
     const { asarPath, filePath } = pathInfo;
 
@@ -705,12 +745,62 @@ export const wrapFsWithAsar = (fs: Record<string, any>) => {
     nextTick(callback!, [null, files]);
   };
 
-  fs.promises.readdir = util.promisify(fs.readdir);
+  const { readdir: readdirPromise } = require('fs').promises;
+  fs.promises.readdir = async function (pathArgument: string, options: ReaddirOptions) {
+    options = getOptions(options);
+    pathArgument = getValidatedPath(pathArgument);
 
-  type ReaddirSyncOptions = { encoding: BufferEncoding | null; withFileTypes?: false };
+    if (options?.recursive != null) {
+      validateBoolean(options?.recursive, 'options.recursive');
+    }
+
+    if (options?.recursive) {
+      return readdirRecursive(pathArgument, options);
+    }
+
+    const pathInfo = splitPath(pathArgument);
+    if (!pathInfo.isAsar) return readdirPromise(pathArgument, options);
+    const { asarPath, filePath } = pathInfo;
+
+    const archive = getOrCreateArchive(asarPath);
+    if (!archive) {
+      return Promise.reject(createError(AsarError.INVALID_ARCHIVE, { asarPath }));
+    }
+
+    const files = archive.readdir(filePath);
+    if (!files) {
+      return Promise.reject(createError(AsarError.NOT_FOUND, { asarPath, filePath }));
+    }
+
+    if (options?.withFileTypes) {
+      const dirents = [];
+      for (const file of files) {
+        const childPath = path.join(filePath, file);
+        const stats = archive.stat(childPath);
+        if (!stats) {
+          throw createError(AsarError.NOT_FOUND, { asarPath, filePath: childPath });
+        }
+        dirents.push(new fs.Dirent(file, stats.type));
+      }
+      return Promise.resolve(dirents);
+    }
+
+    return Promise.resolve(files);
+  };
 
   const { readdirSync } = fs;
-  fs.readdirSync = function (pathArgument: string, options: ReaddirSyncOptions | BufferEncoding | null) {
+  fs.readdirSync = function (pathArgument: string, options: ReaddirOptions) {
+    options = getOptions(options);
+    pathArgument = getValidatedPath(pathArgument);
+
+    if (options?.recursive != null) {
+      validateBoolean(options?.recursive, 'options.recursive');
+    }
+
+    if (options?.recursive) {
+      return readdirSyncRecursive(pathArgument, options);
+    }
+
     const pathInfo = splitPath(pathArgument);
     if (!pathInfo.isAsar) return readdirSync.apply(this, arguments);
     const { asarPath, filePath } = pathInfo;
@@ -725,7 +815,7 @@ export const wrapFsWithAsar = (fs: Record<string, any>) => {
       throw createError(AsarError.NOT_FOUND, { asarPath, filePath });
     }
 
-    if (options && (options as ReaddirSyncOptions).withFileTypes) {
+    if (options?.withFileTypes) {
       const dirents = [];
       for (const file of files) {
         const childPath = path.join(filePath, file);
@@ -741,7 +831,8 @@ export const wrapFsWithAsar = (fs: Record<string, any>) => {
     return files;
   };
 
-  const { internalModuleReadJSON } = internalBinding('fs');
+  const binding = internalBinding('fs');
+  const { internalModuleReadJSON, kUsePromises } = binding;
   internalBinding('fs').internalModuleReadJSON = (pathArgument: string) => {
     const pathInfo = splitPath(pathArgument);
     if (!pathInfo.isAsar) return internalModuleReadJSON(pathArgument);
@@ -787,6 +878,163 @@ export const wrapFsWithAsar = (fs: Record<string, any>) => {
     return (stats.type === AsarFileType.kDirectory) ? 1 : 0;
   };
 
+  async function readdirRecursive (originalPath: string, options: ReaddirOptions) {
+    const result: any[] = [];
+
+    const pathInfo = splitPath(originalPath);
+    let queue: [string, string[]][] = [];
+
+    let initialItem = [];
+    if (pathInfo.isAsar) {
+      const archive = getOrCreateArchive(pathInfo.asarPath);
+      if (!archive) return result;
+      const files = archive.readdir(pathInfo.filePath);
+      if (!files) return result;
+      initialItem = files;
+    } else {
+      initialItem = await binding.readdir(
+        path.toNamespacedPath(originalPath),
+        options!.encoding,
+        !!options!.withFileTypes,
+        kUsePromises
+      );
+    }
+
+    queue = [[originalPath, initialItem]];
+
+    if (options?.withFileTypes) {
+      while (queue.length > 0) {
+        // @ts-expect-error this is a valid array destructure assignment.
+        const { 0: pathArg, 1: readDir } = queue.pop();
+        for (const dirent of getDirents(pathArg, readDir)) {
+          result.push(dirent);
+          if (dirent.isDirectory()) {
+            const direntPath = path.join(pathArg, dirent.name);
+            const info = splitPath(direntPath);
+            let archive;
+            let readdirResult;
+            if (info.isAsar) {
+              archive = getOrCreateArchive(info.asarPath);
+              if (!archive) continue;
+              readdirResult = archive.readdir(info.filePath);
+            } else {
+              readdirResult = await binding.readdir(
+                direntPath,
+                options.encoding,
+                true,
+                kUsePromises
+              );
+            }
+            queue.push([direntPath, readdirResult]);
+          }
+        }
+      }
+    } else {
+      while (queue.length > 0) {
+        // @ts-expect-error this is a valid array destructure assignment.
+        const { 0: pathArg, 1: readDir } = queue.pop();
+        for (const ent of readDir) {
+          const direntPath = path.join(pathArg, ent);
+          const stat = internalBinding('fs').internalModuleStat(direntPath);
+          result.push(path.relative(originalPath, direntPath));
+
+          if (stat === 1) {
+            const subPathInfo = splitPath(direntPath);
+            let item = [];
+            if (subPathInfo.isAsar) {
+              const archive = getOrCreateArchive(subPathInfo.asarPath);
+              if (!archive) return;
+              const files = archive.readdir(subPathInfo.filePath);
+              if (!files) return result;
+              item = files;
+            } else {
+              item = await binding.readdir(
+                path.toNamespacedPath(direntPath),
+                options!.encoding,
+                false,
+                kUsePromises
+              );
+            }
+            queue.push([direntPath, item]);
+          }
+        }
+      }
+    }
+
+    return result;
+  }
+
+  function readdirSyncRecursive (basePath: string, options: ReaddirOptions) {
+    const withFileTypes = Boolean(options!.withFileTypes);
+    const encoding = options!.encoding;
+
+    const readdirResults: string[] = [];
+    const pathsQueue = [basePath];
+
+    function read (pathArg: string) {
+      let readdirResult;
+      const pathInfo = splitPath(pathArg);
+
+      let archive;
+      if (pathInfo.isAsar) {
+        const { asarPath, filePath } = pathInfo;
+        archive = getOrCreateArchive(asarPath);
+        if (!archive) return;
+
+        readdirResult = archive.readdir(filePath);
+      } else {
+        readdirResult = binding.readdir(
+          path.toNamespacedPath(pathArg),
+          encoding,
+          withFileTypes
+        );
+      }
+
+      if (readdirResult === undefined) return;
+
+      if (withFileTypes) {
+        // Calling `readdir` with `withFileTypes=true`, the result is an array of arrays.
+        // The first array is the names, and the second array is the types.
+        // They are guaranteed to be the same length; hence, setting `length` to the length
+        // of the first array within the result.
+        const length = readdirResult[0].length;
+        for (let i = 0; i < length; i++) {
+          let dirent;
+          if (pathInfo.isAsar) {
+            const stats = archive!.stat(pathArg);
+            if (!stats) continue;
+            dirent = getDirent(pathArg, readdirResult[0][i], stats.type);
+          } else {
+            dirent = getDirent(pathArg, readdirResult[0][i], readdirResult[1][i]);
+          }
+
+          readdirResults.push(dirent);
+          if (dirent.isDirectory()) {
+            pathsQueue.push(path.join(dirent.path, dirent.name));
+          }
+        }
+      } else {
+        for (let i = 0; i < readdirResult.length; i++) {
+          const resultPath = path.join(pathArg, readdirResult[i]);
+          const relativeResultPath = path.relative(basePath, resultPath);
+          const stat = internalBinding('fs').internalModuleStat(resultPath);
+          readdirResults.push(relativeResultPath);
+
+          // 1 indicates directory
+          if (stat === 1) {
+            pathsQueue.push(resultPath);
+          }
+        }
+      }
+    }
+
+    for (let i = 0; i < pathsQueue.length; i++) {
+      read(pathsQueue[i]);
+    }
+
+    return readdirResults;
+  }
+
   // Calling mkdir for directory inside asar archive should throw ENOTDIR
   // error, but on Windows it throws ENOENT.
   if (process.platform === 'win32') {

spec/asar-spec.ts

@@ -161,6 +161,7 @@ describe('asar package', function () {
 
       fs = require('node:fs')
       path = require('node:path')
+      fixtures = ${JSON.stringify(fixtures)}
       asarDir = ${JSON.stringify(asarDir)}
 
       // This is used instead of util.promisify for some tests to dodge the
@@ -897,6 +898,37 @@ describe('asar package', function () {
         expect(dirs).to.deep.equal(['dir1', 'dir2', 'dir3', 'file1', 'file2', 'file3', 'link1', 'link2', 'ping.js']);
       });
 
+      itremote('supports recursive readdirSync', async () => {
+        const dir = path.join(fixtures, 'recursive-asar');
+        const files = await fs.readdirSync(dir, { recursive: true });
+        expect(files).to.have.members([
+          'a.asar',
+          'nested',
+          'test.txt',
+          path.join('a.asar', 'dir1'),
+          path.join('a.asar', 'dir2'),
+          path.join('a.asar', 'dir3'),
+          path.join('a.asar', 'file1'),
+          path.join('a.asar', 'file2'),
+          path.join('a.asar', 'file3'),
+          path.join('a.asar', 'link1'),
+          path.join('a.asar', 'link2'),
+          path.join('a.asar', 'ping.js'),
+          path.join('nested', 'hello.txt'),
+          path.join('a.asar', 'dir1', 'file1'),
+          path.join('a.asar', 'dir1', 'file2'),
+          path.join('a.asar', 'dir1', 'file3'),
+          path.join('a.asar', 'dir1', 'link1'),
+          path.join('a.asar', 'dir1', 'link2'),
+          path.join('a.asar', 'dir2', 'file1'),
+          path.join('a.asar', 'dir2', 'file2'),
+          path.join('a.asar', 'dir2', 'file3'),
+          path.join('a.asar', 'dir3', 'file1'),
+          path.join('a.asar', 'dir3', 'file2'),
+          path.join('a.asar', 'dir3', 'file3')
+        ]);
+      });
+
       itremote('reads dirs from a normal dir', function () {
         const p = path.join(asarDir, 'a.asar', 'dir1');
         const dirs = fs.readdirSync(p);
@@ -944,6 +976,37 @@ describe('asar package', function () {
         expect(dirs).to.deep.equal(['dir1', 'dir2', 'dir3', 'file1', 'file2', 'file3', 'link1', 'link2', 'ping.js']);
       });
 
+      itremote('supports recursive readdirSync', async () => {
+        const dir = path.join(fixtures, 'recursive-asar');
+        const files = await promisify(fs.readdir)(dir, { recursive: true });
+        expect(files).to.have.members([
+          'a.asar',
+          'nested',
+          'test.txt',
+          path.join('a.asar', 'dir1'),
+          path.join('a.asar', 'dir2'),
+          path.join('a.asar', 'dir3'),
+          path.join('a.asar', 'file1'),
+          path.join('a.asar', 'file2'),
+          path.join('a.asar', 'file3'),
+          path.join('a.asar', 'link1'),
+          path.join('a.asar', 'link2'),
+          path.join('a.asar', 'ping.js'),
+          path.join('nested', 'hello.txt'),
+          path.join('a.asar', 'dir1', 'file1'),
+          path.join('a.asar', 'dir1', 'file2'),
+          path.join('a.asar', 'dir1', 'file3'),
+          path.join('a.asar', 'dir1', 'link1'),
+          path.join('a.asar', 'dir1', 'link2'),
+          path.join('a.asar', 'dir2', 'file1'),
+          path.join('a.asar', 'dir2', 'file2'),
+          path.join('a.asar', 'dir2', 'file3'),
+          path.join('a.asar', 'dir3', 'file1'),
+          path.join('a.asar', 'dir3', 'file2'),
+          path.join('a.asar', 'dir3', 'file3')
+        ]);
+      });
+
       itremote('supports withFileTypes', async () => {
         const p = path.join(asarDir, 'a.asar');
 
@@ -1008,6 +1071,37 @@ describe('asar package', function () {
         expect(dirs).to.deep.equal(['dir1', 'dir2', 'dir3', 'file1', 'file2', 'file3', 'link1', 'link2', 'ping.js']);
       });
 
+      itremote('supports recursive readdir', async () => {
+        const dir = path.join(fixtures, 'recursive-asar');
+        const files = await fs.promises.readdir(dir, { recursive: true });
+        expect(files).to.have.members([
+          'a.asar',
+          'nested',
+          'test.txt',
+          path.join('a.asar', 'dir1'),
+          path.join('a.asar', 'dir2'),
+          path.join('a.asar', 'dir3'),
+          path.join('a.asar', 'file1'),
+          path.join('a.asar', 'file2'),
+          path.join('a.asar', 'file3'),
+          path.join('a.asar', 'link1'),
+          path.join('a.asar', 'link2'),
+          path.join('a.asar', 'ping.js'),
+          path.join('nested', 'hello.txt'),
+          path.join('a.asar', 'dir1', 'file1'),
+          path.join('a.asar', 'dir1', 'file2'),
+          path.join('a.asar', 'dir1', 'file3'),
+          path.join('a.asar', 'dir1', 'link1'),
+          path.join('a.asar', 'dir1', 'link2'),
+          path.join('a.asar', 'dir2', 'file1'),
+          path.join('a.asar', 'dir2', 'file2'),
+          path.join('a.asar', 'dir2', 'file3'),
+          path.join('a.asar', 'dir3', 'file1'),
+          path.join('a.asar', 'dir3', 'file2'),
+          path.join('a.asar', 'dir3', 'file3')
+        ]);
+      });
+
       itremote('supports withFileTypes', async function () {
         const p = path.join(asarDir, 'a.asar');
         const dirs = await fs.promises.readdir(p, { withFileTypes: true });

spec/fixtures/recursive-asar/nested/hello.txt

@@ -0,0 +1 @@
+goodbye!

spec/fixtures/recursive-asar/test.txt

@@ -0,0 +1 @@
+Hello!