fix: ts-node --esm on Node v20

When running `ts-node --esm`, a child process is spawned with the
`child-loader.mjs` loader, `dist/child/child-entrypoint.js` main,
and `argv[2]` set to the base64 encoded compressed configuration
payload.

`child-loader.mjs` imports and re-exports the functions defined
in `src/child/child-loader.ts`. These are initially set to empty
loader hooks which call the next hook in line until they are
defined by calling `lateBindHooks()`.

`child-entrypoint.ts` reads the config payload from argv, and
bootstraps the registration process, which then calls
`lateBindHooks()`.

Presumably, the reason for this hand-off is because `--loader`
hooks do not have access to `process.argv`.  Unfortunately, in
node 20, they don't have access to anything else, either, so
calling `lateBindHooks` is effectively a no-op; the
`child-loader.ts` where the hooks end up getting bound is not the
same one that is being used as the actual loader.

To solve this, the following changes are added:

1. An `isLoaderThread` flag is added to the BootstrapState. If
   this flag is set, then no further processing is performed
   beyond binding the loader hooks.
2. `callInChild` adds the config payload to _both_ the argv and
   the loader URL as a query param.
3. In the `child-loader.mjs` loader, only on node v20 and higher,
   the config payload is read from `import.meta.url`, and
   `bootstrap` is called, setting the `isLoaderThread` flag.

I'm not super enthusiastic about this implementation. It
definitely feels like there's a refactoring opportunity to clean
it up, as it adds some copypasta between child-entrypoint.ts and
child-loader.mjs. A further improvement would be to remove the
late-binding handoff complexity entirely, and _always_ pass the
config payload on the loader URL rather than on process.argv.

Author: isaacs

child-loader.mjs

@@ -1,8 +1,11 @@
-import { fileURLToPath } from 'url';
 import { createRequire } from 'module';
+import { fileURLToPath } from 'url';
+
 const require = createRequire(fileURLToPath(import.meta.url));
 
 // TODO why use require() here?  I think we can just `import`
 /** @type {import('./dist/child-loader')} */
 const childLoader = require('./dist/child/child-loader');
-export const { resolve, load, getFormat, transformSource } = childLoader;
+export const { resolve, load, getFormat, transformSource, bindFromLoaderThread } = childLoader;
+
+bindFromLoaderThread(import.meta.url);

esm.mjs

@@ -1,22 +1,7 @@
 import { fileURLToPath } from 'url';
 import { createRequire } from 'module';
-import { versionGteLt } from './dist/util.js';
 const require = createRequire(fileURLToPath(import.meta.url));
 
 /** @type {import('./dist/esm')} */
 const esm = require('./dist/esm');
-export const { resolve, load, getFormat, transformSource } = esm.registerAndCreateEsmHooks();
-
-// Affordance for node 20, where load() happens in an isolated thread
-const offThreadLoader = versionGteLt(process.versions.node, '20.0.0');
-export const globalPreload = () => {
-  if (!offThreadLoader) {
-    return '';
-  }
-  const self = fileURLToPath(import.meta.url);
-  return `
-const { createRequire } = getBuiltin('module');
-const require = createRequire(${JSON.stringify(self)});
-require('./dist/esm').registerAndCreateEsmHooks();
-`;
-};
+export const { resolve, load, getFormat, transformSource, globalPreload } = esm.registerAndCreateEsmHooks();

src/bin.ts

@@ -71,6 +71,7 @@ export interface BootstrapState {
   parseArgvResult: ReturnType<typeof parseArgv>;
   phase2Result?: ReturnType<typeof phase2>;
   phase3Result?: ReturnType<typeof phase3>;
+  isLoaderThread?: boolean;
 }
 
 /** @internal */
@@ -441,7 +442,7 @@ function getEntryPointInfo(state: BootstrapState) {
 }
 
 function phase4(payload: BootstrapState) {
-  const { isInChildProcess, tsNodeScript } = payload;
+  const { isInChildProcess, tsNodeScript, isLoaderThread } = payload;
   const { version, showConfig, restArgs, code, print, argv } = payload.parseArgvResult;
   const { cwd } = payload.phase2Result!;
   const { preloadedConfig } = payload.phase3Result!;
@@ -522,8 +523,12 @@ function phase4(payload: BootstrapState) {
 
   if (replStuff) replStuff.state.path = join(cwd, REPL_FILENAME(service.ts.version));
 
-  if (isInChildProcess)
+  if (isInChildProcess) {
     (require('./child/child-loader') as typeof import('./child/child-loader')).lateBindHooks(createEsmHooks(service));
+    // we should not do anything else at this point in the loader thread,
+    // let the entrypoint run the actual program.
+    if (isLoaderThread) return;
+  }
 
   // Bind REPL service to ts-node compiler service (chicken-and-egg problem)
   replStuff?.repl.setService(service);

src/child/child-loader.ts

@@ -1,5 +1,24 @@
 import type { NodeLoaderHooksAPI1, NodeLoaderHooksAPI2 } from '..';
 import { filterHooksByAPIVersion } from '../esm';
+import { URL } from 'url';
+import { bootstrap } from '../bin';
+import { versionGteLt } from '../util';
+import { argPrefix, decompress } from './argv-payload';
+
+// On node v20, we cannot lateBind the hooks from outside the loader thread
+// so it has to be done in the loader thread.
+export function bindFromLoaderThread(loaderURL: string) {
+  // If we aren't in a loader thread, then skip this step.
+  if (!versionGteLt(process.versions.node, '20.0.0')) return;
+
+  const url = new URL(loaderURL);
+  const base64Payload = url.searchParams.get(argPrefix);
+  if (!base64Payload) throw new Error('unexpected loader url');
+  const state = decompress(base64Payload);
+  state.isInChildProcess = true;
+  state.isLoaderThread = true;
+  bootstrap(state);
+}
 
 let hooks: NodeLoaderHooksAPI1 & NodeLoaderHooksAPI2;
 

src/child/spawn-child.ts

@@ -10,16 +10,20 @@ import { argPrefix, compress } from './argv-payload';
  *   the child process.
  */
 export function callInChild(state: BootstrapState) {
+  const loaderURL = pathToFileURL(require.resolve('../../child-loader.mjs'));
+  const compressedState = compress(state);
+  loaderURL.searchParams.set(argPrefix, compressedState);
+
   const child = spawn(
     process.execPath,
     [
       '--require',
       require.resolve('./child-require.js'),
       '--loader',
       // Node on Windows doesn't like `c:\` absolute paths here; must be `file:///c:/`
-      pathToFileURL(require.resolve('../../child-loader.mjs')).toString(),
+      loaderURL.toString(),
       require.resolve('./child-entrypoint.js'),
-      `${argPrefix}${compress(state)}`,
+      `${argPrefix}${compressedState}`,
       ...state.parseArgvResult.restArgs,
     ],
     {

src/esm.ts

@@ -43,6 +43,7 @@ export namespace NodeLoaderHooksAPI1 {
 export interface NodeLoaderHooksAPI2 {
   resolve: NodeLoaderHooksAPI2.ResolveHook;
   load: NodeLoaderHooksAPI2.LoadHook;
+  globalPreload?: NodeLoaderHooksAPI2.GlobalPreload;
 }
 export namespace NodeLoaderHooksAPI2 {
   export type ResolveHook = (
@@ -74,6 +75,7 @@ export namespace NodeLoaderHooksAPI2 {
   export interface NodeImportAssertions {
     type?: 'json';
   }
+  export type GlobalPreload = () => string;
 }
 
 export type NodeLoaderHooksFormat = 'builtin' | 'commonjs' | 'dynamic' | 'json' | 'module' | 'wasm';
@@ -111,14 +113,24 @@ export function createEsmHooks(tsNodeService: Service) {
   const nodeResolveImplementation = tsNodeService.getNodeEsmResolver();
   const nodeGetFormatImplementation = tsNodeService.getNodeEsmGetFormat();
   const extensions = tsNodeService.extensions;
+  const useLoaderThread = versionGteLt(process.versions.node, '20.0.0');
 
   const hooksAPI = filterHooksByAPIVersion({
     resolve,
     load,
     getFormat,
     transformSource,
+    globalPreload: useLoaderThread ? globalPreload : undefined,
   });
 
+  function globalPreload() {
+    return `
+      const { createRequire } = getBuiltin('module');
+      const require = createRequire(${JSON.stringify(__filename)});
+      require('./index').register();
+    `;
+  }
+
   function isFileUrlOrNodeStyleSpecifier(parsed: UrlWithStringQuery) {
     // We only understand file:// URLs, but in node, the specifier can be a node-style `./foo` or `foo`
     const { protocol } = parsed;