add Node.js v20 support (#28)

* add Node.js v20 support

Since Node.js v20 moves loaders to a separate thread, we can no longer
depend on loading the modules in the loader to get the exports. This is
needed to add our mutable proxy. For Node.js 20, exports are now
retrieved via parsing.

To reduce startup overhead on older versions of Node.js, the previous
method of getting exports is used, avoiding loading and using parsers.

* make stacked loaders work

* restrict get-esm-exports test to node 20

* remove 'unsupported'

* normalize file URLs to paths (important on windows)

* remove trailing \r in esm exports test

Author: bengl

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [12.x, 14.x, 16.10.0, 16.16.0, 16.17.0, 16.x, 17.x, 18.5.0, 18.x]
+        node-version: [12.x, 14.x, 16.10.0, 16.16.0, 16.17.0, 16.x, 17.x, 18.5.0, 18.x, 20.x]
 
     steps:
       - uses: actions/checkout@v2
@@ -23,28 +23,12 @@ jobs:
       - run: npm run test:ts
         if: (matrix.node-version != '12.x' && matrix.node-version != '14.x' && matrix.node-version != '16.10.0')
 
-  build-unsupported:
-    runs-on: ubuntu-latest
-
-    strategy:
-      matrix:
-        node-version: [20.x]
-
-    steps:
-      - uses: actions/checkout@v2
-      - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v2
-        with:
-          node-version: ${{ matrix.node-version }}
-      - run: npm install
-      - run: npm test:unsupported
-
   build-win:
       runs-on: windows-latest
 
       strategy:
         matrix:
-          node-version: [ 12.x, 14.x, 16.10.0, 16.16.0, 16.17.0, 16.x, 18.5.0, 18.x ]
+          node-version: [12.x, 14.x, 16.10.0, 16.16.0, 16.17.0, 16.x, 18.5.0, 18.x, 20.x]
 
       steps:
         - uses: actions/checkout@v2

hook.js

@@ -14,6 +14,12 @@ const NODE_MINOR = Number(NODE_VERSION[1])
 
 let entrypoint
 
+if (NODE_MAJOR >= 20) {
+  getExports = require('./lib/get-exports.js')
+} else {
+  getExports = (url) => import(url).then(Object.keys)
+}
+
 function hasIitm (url) {
   try {
     return new URL(url).searchParams.has('iitm')
@@ -103,16 +109,16 @@ function createHook (meta) {
 
     return {
       url: addIitm(url.url),
-      shortCircuit: true
+      shortCircuit: true,
+      format: url.format
     }
   }
 
   const iitmURL = new URL('lib/register.js', meta.url).toString()
   async function getSource (url, context, parentGetSource) {
     if (hasIitm(url)) {
       const realUrl = deleteIitm(url)
-      const realModule = await import(realUrl)
-      const exportNames = Object.keys(realModule)
+      const exportNames = await getExports(realUrl, context, parentGetSource)
       return {
         source: `
 import { register } from '${iitmURL}'
@@ -137,7 +143,7 @@ register('${realUrl}', namespace, set, '${specifiers.get(realUrl)}')
   // For Node.js 16.12.0 and higher.
   async function load (url, context, parentLoad) {
     if (hasIitm(url)) {
-      const { source } = await getSource(url, context)
+      const { source } = await getSource(url, context, parentLoad)
       return {
         source,
         shortCircuit: true,
@@ -148,10 +154,7 @@ register('${realUrl}', namespace, set, '${specifiers.get(realUrl)}')
     return parentLoad(url, context, parentLoad)
   }
 
-  if (NODE_MAJOR >= 20) {
-    process.emitWarning('import-in-the-middle is currently unsupported on Node.js v20 and has been disabled.')
-    return {} // TODO: Add support for Node >=20
-  } else if (NODE_MAJOR >= 17 || (NODE_MAJOR === 16 && NODE_MINOR >= 12)) {
+  if (NODE_MAJOR >= 17 || (NODE_MAJOR === 16 && NODE_MINOR >= 12)) {
     return { load, resolve }
   } else {
     return {

lib/get-esm-exports.js

@@ -0,0 +1,97 @@
+'use strict'
+
+const { Parser } = require('acorn')
+const { importAssertions } = require('acorn-import-assertions');
+
+const acornOpts = {
+  ecmaVersion: 'latest',
+  sourceType: 'module'
+}
+
+const parser = Parser.extend(importAssertions)
+
+function warn (txt) {
+  process.emitWarning(txt, 'get-esm-exports')
+}
+
+function getEsmExports (moduleStr) {
+  const exportedNames = new Set()
+  const tree = parser.parse(moduleStr, acornOpts)
+  for (const node of tree.body) {
+    if (!node.type.startsWith('Export')) continue
+    switch (node.type) {
+      case 'ExportNamedDeclaration':
+        if (node.declaration) {
+          parseDeclaration(node, exportedNames)
+        } else {
+          parseSpecifiers(node, exportedNames)
+        }
+        break
+      case 'ExportDefaultDeclaration':
+        exportedNames.add('default')
+        break
+      case 'ExportAllDeclaration':
+        if (node.exported) {
+          exportedNames.add(node.exported.name)
+        } else {
+          exportedNames.add('*')
+        }
+        break
+      default:
+        warn('unrecognized export type: ' + node.type)
+    }
+  }
+  return Array.from(exportedNames)
+}
+
+function parseDeclaration (node, exportedNames) {
+  switch (node.declaration.type) {
+    case 'FunctionDeclaration':
+      exportedNames.add(node.declaration.id.name)
+      break
+    case 'VariableDeclaration':
+      for (const varDecl of node.declaration.declarations) {
+        parseVariableDeclaration(varDecl, exportedNames)
+      }
+      break
+    case 'ClassDeclaration':
+      exportedNames.add(node.declaration.id.name)
+      break
+    default:
+      warn('unknown declaration type: ' + node.delcaration.type)
+  }
+}
+
+function parseVariableDeclaration (node, exportedNames) {
+  switch (node.id.type) {
+    case 'Identifier':
+      exportedNames.add(node.id.name)
+      break
+    case 'ObjectPattern':
+      for (const prop of node.id.properties) {
+        exportedNames.add(prop.value.name)
+      }
+      break
+    case 'ArrayPattern':
+      for (const elem of node.id.elements) {
+        exportedNames.add(elem.name)
+      }
+      break
+    default:
+      warn('unknown variable declaration type: ' + node.id.type)
+  }
+}
+
+function parseSpecifiers (node, exportedNames) {
+  for (const specifier of node.specifiers) {
+    if (specifier.exported.type === 'Identifier') {
+      exportedNames.add(specifier.exported.name)
+    } else if (specifier.exported.type === 'Literal') {
+      exportedNames.add(specifier.exported.value)
+    } else {
+      warn('unrecognized specifier type: ' + specifier.exported.type)
+    }
+  }
+}
+
+module.exports = getEsmExports

lib/get-exports.js

@@ -0,0 +1,51 @@
+'use strict'
+
+const getEsmExports = require('./get-esm-exports.js')
+const { parse: getCjsExports } = require('cjs-module-lexer')
+const fs = require('fs')
+const { fileURLToPath } = require('url')
+
+function addDefault(arr) {
+  return Array.from(new Set(['default', ...arr]))
+}
+
+async function getExports (url, context, parentLoad) {
+  // `parentLoad` gives us the possibility of getting the source
+  // from an upstream loader. This doesn't always work though,
+  // so later on we fall back to reading it from disk.
+  const parentCtx = await parentLoad(url, context)
+  let source = parentCtx.source
+  const format = parentCtx.format
+
+  // TODO support non-node/file urls somehow?
+  if (format === 'builtin') {
+    // Builtins don't give us the source property, so we're stuck
+    // just requiring it to get the exports.
+    return addDefault(Object.keys(require(url)))
+  }
+
+  if (!source) {
+    // Sometimes source is retrieved by parentLoad, sometimes it isn't.
+    source = fs.readFileSync(fileURLToPath(url), 'utf8')
+  }
+
+  if (format === 'module') {
+    return getEsmExports(source)
+  }
+  if (format === 'commonjs') {
+    return addDefault(getCjsExports(source).exports)
+  }
+
+  // At this point our `format` is either undefined or not known by us. Fall
+  // back to parsing as ESM/CJS.
+  const esmExports = getEsmExports(source)
+  if (!esmExports.length) {
+    // TODO(bengl) it's might be possible to get here if somehow the format
+    // isn't set at first and yet we have an ESM module with no exports.
+    // I couldn't construct an example that would do this, so maybe it's
+    // impossible?
+    return addDefault(getCjsExports(source).exports)
+  }
+}
+
+module.exports = getExports

package.json

@@ -4,12 +4,11 @@
   "description": "Intercept imports in Node.js",
   "main": "index.js",
   "scripts": {
-    "test": "c8 --check-coverage --lines 85 imhotap --runner test/runtest --files test/{hook,low-level,other}/*",
-    "test:unsupported": "imhotap --runner test/runtest --files test/hook/loader.mjs",
-    "test-win": "c8 --check-coverage --lines 85 imhotap --runner test\\runtest.bat --files test/{hook,low-level,other}/*",
+    "test": "c8 --check-coverage --lines 85 imhotap --runner test/runtest --files test/{hook,low-level,other,get-esm-exports}/*",
+    "test-win": "c8 --check-coverage --lines 85 imhotap --runner test\\runtest.bat --files test/{hook,low-level,other,get-esm-exports}/*",
     "test:ts": "c8 imhotap --runner test/runtest --files test/typescript/*.test.mts",
     "test-win:ts": "c8 imhotap --runner test\\runtest.bat --files test/typescript/*.test.mts",
-    "coverage": "c8 --reporter html imhotap --runner test/runtest --files test/{hook,low-level,other}/* && echo '\nNow open coverage/index.html\n'"
+    "coverage": "c8 --reporter html imhotap --runner test/runtest --files test/{hook,low-level,other,get-esm-exports}/* && echo '\nNow open coverage/index.html\n'"
   },
   "repository": {
     "type": "git",
@@ -37,6 +36,9 @@
     "typescript": "^4.7.4"
   },
   "dependencies": {
+    "acorn": "^8.8.2",
+    "acorn-import-assertions": "^1.9.0",
+    "cjs-module-lexer": "^1.2.2",
     "module-details-from-path": "^1.0.3"
   }
 }

test/fixtures/esm-exports.txt

@@ -0,0 +1,32 @@
+// Exporting declarations
+export let name1, name2/*, … */; // also var //| name1,name2
+export const name1 = 1, name2 = 2/*, … */; // also var, let //| name1,name2
+export function functionName() { /* … */ } //| functionName
+export class ClassName { /* … */ } //| ClassName
+export function* generatorFunctionName() { /* … */ } //| generatorFunctionName
+export const { name1, name2: bar } = o; //| name1,bar
+export const [ name1, name2 ] = array; //| name1,name2
+
+// Export list
+let name1, nameN; export { name1, /* …, */ nameN }; //| name1,nameN
+let variable1, variable2, nameN; export { variable1 as name1, variable2 as name2, /* …, */ nameN }; //| name1,name2,nameN
+let variable1; export { variable1 as "string name" }; //| string name
+let name1; export { name1 as default /*, … */ }; //| default
+
+// Default exports
+export default expression; //| default
+export default function functionName() { /* … */ } //| default
+export default class ClassName { /* … */ } //| default
+export default function* generatorFunctionName() { /* … */ } //| default
+export default function () { /* … */ } //| default
+export default class { /* … */ } //| default
+export default function* () { /* … */ } //| default
+
+// Aggregating modules
+export * from "module-name"; //| *
+export * as name1 from "module-name"; //| name1
+export { name1, /* …, */ nameN } from "module-name"; //| name1,nameN
+export { import1 as name1, import2 as name2, /* …, */ nameN } from "module-name"; //| name1,name2,nameN
+export { default, /* …, */ } from "module-name"; //| default
+export { default as name1 } from "module-name"; //| name1
+

test/get-esm-exports/v20-get-esm-exports.js

@@ -0,0 +1,31 @@
+'use strict'
+
+const getEsmExports = require('../../lib/get-esm-exports.js')
+const fs = require('fs')
+const assert = require('assert')
+const path = require('path')
+
+const fixturePath = path.join(__dirname, '../fixtures/esm-exports.txt')
+const fixture = fs.readFileSync(fixturePath, 'utf8')
+
+fixture.split('\n').forEach(line => {
+  if (!line.includes(' //| ')) return
+  const [mod, testStr] = line.split(' //| ')
+  const expectedNames = testStr.split(',').map(x => x.trim())
+  if (expectedNames[0] === '') {
+    expectedNames.length = 0
+  }
+  const names = getEsmExports(mod)
+  assert.deepEqual(expectedNames, names)
+  console.log(`${mod}\n  ✅ contains exports: ${testStr}`)
+})
+
+// // Generate fixture data
+// fixture.split('\n').forEach(line => {
+//   if (!line.includes('export ')) {
+//     console.log(line)
+//     return
+//   }
+//   const names = getEsmExports(line)
+//   console.log(line, '//|', names.join(','))
+// })