vfs: integrate with CJS and ESM module loaders

benchmark/vfs/module-graph.js

@@ -0,0 +1,62 @@
+'use strict';
+const path = require('path');
+const { pathToFileURL } = require('url');
+const common = require('../common.js');
+
+const bench = common.createBenchmark(main, {
+  type: ['cjs', 'esm'],
+  files: [1e2, 1e3],
+  n: [10],
+}, { flags: ['--experimental-vfs', '--no-warnings'] });
+
+// Builds a module graph of `files` packages, each with its own package.json,
+// an index requiring a package-local file and a shared root module, and an
+// entry point that pulls in every package.
+function buildGraph(layer, files, type) {
+  const entryRequires = [];
+  if (type === 'esm') {
+    layer.writeFileSync('/package.json', '{"type":"module"}');
+  }
+  layer.writeFileSync('/shared.js',
+                      type === 'cjs' ? 'module.exports = 0;' : 'export default 0;');
+  for (let i = 0; i < files; i++) {
+    layer.mkdirSync(`/${i}`, { recursive: true });
+    if (type === 'cjs') {
+      layer.writeFileSync(`/${i}/package.json`, '{"main":"index.js"}');
+      layer.writeFileSync(`/${i}/lib.js`, 'module.exports = 1;');
+      layer.writeFileSync(
+        `/${i}/index.js`,
+        'require("./lib.js"); require("../shared.js"); module.exports = __filename;');
+      entryRequires.push(`require('./${i}/');`);
+    } else {
+      layer.writeFileSync(`/${i}/package.json`, '{"type":"module"}');
+      layer.writeFileSync(`/${i}/lib.js`, 'export default 1;');
+      layer.writeFileSync(
+        `/${i}/index.js`,
+        'import "./lib.js"; import "../shared.js"; export default import.meta.url;');
+      entryRequires.push(`import './${i}/index.js';`);
+    }
+  }
+  layer.writeFileSync('/entry.js', entryRequires.join('\n'));
+}
+
+async function main({ n, type, files }) {
+  const vfs = require('node:vfs');
+  const layer = vfs.create();
+  buildGraph(layer, files, type);
+
+  bench.start();
+  for (let i = 0; i < n; i++) {
+    const mountPoint = layer.mount();
+    const entry = path.join(mountPoint, 'entry.js');
+    if (type === 'cjs') {
+      require(entry);
+    } else {
+      await import(pathToFileURL(entry).href);
+    }
+    // Unmounting purges the module caches for the mount prefix, so every
+    // iteration is a cold load of the full graph.
+    layer.unmount();
+  }
+  bench.end(n * files);
+}

doc/api/vfs.md

@@ -61,6 +61,11 @@ callback-based, and promise-based file system methods that mirror the
 shape of the [`node:fs`][] API. All paths are POSIX-style and absolute
 (starting with `/`).
 
+By default, the file tree is private to the VFS instance. To expose
+it through the global `node:fs` module, `require()`, and `import`,
+call [`vfs.mount()`][]; call [`vfs.unmount()`][] (or rely on a
+`using` declaration) to detach again.
+
 ## `vfs.create([provider][, options])`
 
 <!-- YAML
@@ -107,6 +112,124 @@ added: v26.4.0
   * `emitExperimentalWarning` {boolean} Whether to emit the experimental
     warning. **Default:** `true`.
 
+### `vfs.mount()`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* Returns: {string} The absolute mount point.
+
+Mounts the virtual file system and returns the resulting mount point.
+After mounting, files in the VFS can be accessed through the
+`node:fs` module and resolved through `require()` and `import`
+using paths under the returned mount point.
+
+Mount points always live inside a reserved namespace that cannot have child file system entries,
+so virtual paths never conflate with (or shadow) real paths. The virtual path scheme is subject to
+change and users should not manually construct them based on assumptions. Instead, obtain
+them from what `vfs.mount()` returns or `vfs.mountPoint`.
+
+```cjs
+const vfs = require('node:vfs');
+const fs = require('node:fs');
+
+const myVfs = vfs.create();
+myVfs.writeFileSync('/data.txt', 'Hello');
+const mountPoint = myVfs.mount();
+// e.g. '/dev/null/vfs/0'
+
+fs.readFileSync(`${mountPoint}/data.txt`, 'utf8'); // 'Hello'
+```
+
+Each `VirtualFileSystem` instance may be mounted at most once at a
+time. Attempting to mount an already-mounted instance throws
+`ERR_INVALID_STATE`. Because each instance mounts inside its own
+per-layer namespace, mounts from different instances can never
+overlap.
+
+The VFS supports the [Explicit Resource Management][] proposal. Use
+a `using` declaration to unmount automatically when leaving scope:
+
+```cjs
+const vfs = require('node:vfs');
+const fs = require('node:fs');
+
+let mountPoint;
+{
+  using myVfs = vfs.create();
+  myVfs.writeFileSync('/data.txt', 'Hello');
+  mountPoint = myVfs.mount();
+
+  fs.readFileSync(`${mountPoint}/data.txt`, 'utf8'); // 'Hello'
+} // VFS is automatically unmounted here
+
+fs.existsSync(`${mountPoint}/data.txt`); // false
+```
+
+### `vfs.unmount()`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+Unmounts the virtual file system. After unmounting, virtual files
+are no longer reachable through `node:fs`, `require()`, or `import`.
+The same instance may be mounted again by calling `mount()`.
+
+This method is idempotent: calling `unmount()` on a VFS that is not
+currently mounted has no effect.
+
+### `vfs.mounted`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* {boolean}
+
+`true` while the VFS is mounted; `false` otherwise.
+
+### `vfs.mountPoint`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* {string | null}
+
+The current mount point as an absolute string (the value returned by
+the last [`vfs.mount()`][] call), or `null` when the VFS is not
+mounted.
+
+### `vfs.mountPointURL`
+
+<!-- YAML
+added: REPLACEME
+-->
+
+* {string | null}
+
+The current mount point as a `file:` URL string (the [`vfs.mountPoint`][]
+path converted with [`url.pathToFileURL()`][]), or `null` when the VFS
+is not mounted.
+
+This is a convenience for addressing mounted files with URL-based
+APIs such as dynamic `import()`:
+
+```mjs
+import vfs from 'node:vfs';
+
+const myVfs = vfs.create();
+myVfs.writeFileSync('/mod.mjs', 'export const value = 42;');
+myVfs.mount();
+
+const { value } = await import(`${myVfs.mountPointURL}/mod.mjs`);
+console.log(value); // 42
+
+myVfs.unmount();
+```
+
 ### `vfs.provider`
 
 <!-- YAML
@@ -196,6 +319,104 @@ The promise namespace mirrors `fs.promises` and includes `readFile`,
 `access`, `rm`, `truncate`, `link`, `mkdtemp`, `chmod`, `chown`, `lchown`,
 `utimes`, `lutimes`, `open`, `lchmod`, and `watch`.
 
+## Module loader integration
+
+Once a `VirtualFileSystem` is mounted, paths under the mount point
+participate in module resolution and loading. The [CommonJS
+resolution algorithm][] used by [`require()`][] and
+[`require.resolve()`][] and the [ES modules resolution algorithm][]
+used by `import` and [`import.meta.resolve()`][] are unchanged;
+instead, every file system operation those algorithms perform is
+dispatched on the path being probed: paths under a mount point are
+served by the owning VFS, and all other paths are served by the real
+file system. Files served from the VFS therefore behave as
+first-class modules.
+
+Because mounted paths live in a reserved namespace that cannot exist
+on disk, any given path is served either by exactly one VFS or by
+the real file system, never both. There is no search order or
+fallback between the two: if a path under a mount point does not
+exist in the VFS, resolution fails with `ENOENT` without consulting
+the disk, and a mounted layer never shadows a real directory.
+
+For resolution purposes the mount point behaves as a file system
+root: `package.json` scope lookups and [loading from `node_modules`
+folders][] stop at the mount point. For example, when
+`${mountPoint}/foo/bar/main.cjs` calls `require('baz')`, the lookup
+goes through:
+
+* `${mountPoint}/foo/bar/node_modules/baz`
+* `${mountPoint}/foo/node_modules/baz`
+* `${mountPoint}/node_modules/baz`
+* If `$NODE_PATH` is set, the folders listed in `$NODE_PATH`
+* `$HOME/.node_modules/baz`
+* `$HOME/.node_libraries/baz`
+* `$PREFIX/lib/node/baz`
+
+The last four entries are [the global folders][], which are legacy
+CommonJS behavior and do not apply to `import`. Absolute specifiers
+may cross the boundary in either direction: a module on the real
+file system can `require()` a mounted path, and a virtual module can
+`require()` a real one.
+
+```cjs
+const vfs = require('node:vfs');
+
+const myVfs = vfs.create();
+myVfs.mkdirSync('/lib');
+myVfs.writeFileSync('/lib/greet.js', 'module.exports = () => "hi";');
+myVfs.writeFileSync(
+  '/lib/package.json', '{"main": "./greet.js"}');
+const mountPoint = myVfs.mount();
+
+const greet = require(`${mountPoint}/lib`);
+console.log(greet()); // 'hi'
+
+myVfs.unmount();
+```
+
+For ECMAScript modules, use `file:` URLs when passing mounted paths
+to dynamic `import()`. [`vfs.mountPointURL`][] provides the mount
+point in that form; this keeps VFS imports portable on Windows,
+where mounted paths use Windows path syntax.
+
+```mjs
+import vfs from 'node:vfs';
+
+const myVfs = vfs.create();
+myVfs.writeFileSync('/mod.mjs', 'export const value = 42;');
+myVfs.mount();
+
+const { value } = await import(`${myVfs.mountPointURL}/mod.mjs`);
+console.log(value); // 42
+
+myVfs.unmount();
+```
+
+CommonJS modules loaded from a mounted VFS are identified by their VFS paths
+that start with the mount point. This is reflected in, for example, `__filename` and
+`__dirname` in the module, or the errors stack traces involving functions from
+the VFS modules. ES modules in the VFS are similarly identified by the `file:` URL of
+their VFS paths and this is reflected in e.g. `import.meta.url`.
+
+Like modules loaded from the real file system, modules loaded from the VFS are
+cached on the first load. When `require()` or `import()` is used to load an absolute
+path or URL that falls under the mounted VFS multiple times, the module is only loaded
+once and subsequent calls return the same instance.
+
+Calling [`vfs.unmount()`][] invalidates the modules that were loaded
+from the mount point: a subsequent `require()` or `import` of a path
+under a re-created mount re-reads the file from the newly mounted
+VFS rather than returning a stale module. Modules loaded from other
+VFS instances or from the real file system are unaffected.
+
+Mounting and unmounting do not stop any module execution that is
+already started, or invalidate any objects materialized from VFS
+modules that are already executed. As with modules in the real file
+system, the callers are responsible for avoiding removal or
+invalidation of modules in the virtual file system while they are
+being loaded.
+
 ## Class: `VirtualProvider`
 
 <!-- YAML
@@ -316,10 +537,23 @@ fields use synthetic but stable values:
 * `blocks` is `Math.ceil(size / 512)`.
 * Times default to the moment the entry was created/last modified.
 
+[CommonJS resolution algorithm]: modules.md#all-together
+[ES modules resolution algorithm]: esm.md#resolution-algorithm
+[Explicit Resource Management]: https://github.com/tc39/proposal-explicit-resource-management
 [`MemoryProvider`]: #class-memoryprovider
 [`RealFSProvider`]: #class-realfsprovider
 [`VirtualFileSystem`]: #class-virtualfilesystem
 [`VirtualProvider`]: #class-virtualprovider
 [`fs.BigIntStats`]: fs.md#class-fsstats
 [`fs.Stats`]: fs.md#class-fsstats
+[`import.meta.resolve()`]: esm.md#importmetaresolvespecifier
 [`node:fs`]: fs.md
+[`require()`]: modules.md#requireid
+[`require.resolve()`]: modules.md#requireresolverequest-options
+[`url.pathToFileURL()`]: url.md#urlpathtofileurlpath-options
+[`vfs.mount()`]: #vfsmount
+[`vfs.mountPointURL`]: #vfsmountpointurl
+[`vfs.mountPoint`]: #vfsmountpoint
+[`vfs.unmount()`]: #vfsunmount
+[loading from `node_modules` folders]: modules.md#loading-from-node_modules-folders
+[the global folders]: modules.md#loading-from-the-global-folders

lib/fs.js

@@ -2105,8 +2105,13 @@ function fstatSync(fd, options = { __proto__: null, bigint: false }) {
 function lstatSync(path, options = { __proto__: null, bigint: false, throwIfNoEntry: true }) {
   const h = vfsState.handlers;
   if (h !== null) {
-    const result = h.lstatSync(path, options);
-    if (result !== undefined) return result;
+    try {
+      const result = h.lstatSync(path, options);
+      if (result !== undefined) return result;
+    } catch (err) {
+      if (err?.code === 'ENOENT' && options?.throwIfNoEntry === false) return;
+      throw err;
+    }
   }
   path = getValidatedPath(path);
   if (permission.isEnabled() && !permission.has('fs.read', path)) {
@@ -2139,8 +2144,13 @@ function lstatSync(path, options = { __proto__: null, bigint: false, throwIfNoEn
 function statSync(path, options = { __proto__: null, bigint: false, throwIfNoEntry: true }) {
   const h = vfsState.handlers;
   if (h !== null) {
-    const result = h.statSync(path, options);
-    if (result !== undefined) return result;
+    try {
+      const result = h.statSync(path, options);
+      if (result !== undefined) return result;
+    } catch (err) {
+      if (err?.code === 'ENOENT' && options?.throwIfNoEntry === false) return undefined;
+      throw err;
+    }
   }
   const stats = binding.stat(
     getValidatedPath(path),