fix: properly collect node_modules when they're ESM and we're node>=16 (#9380)

Author: mmaietta

.changeset/chatty-wolves-live.md

@@ -0,0 +1,7 @@
+---
+"app-builder-lib": patch
+"builder-util": patch
+"dmg-builder": patch
+---
+
+fix: properly collect node_modules when they're ESM and we're node>=16

.github/workflows/test.yaml

@@ -243,6 +243,8 @@ jobs:
         env:
           TEST_FILES: ${{ matrix.testFiles }}
           FORCE_COLOR: 1
+          TEST_SEQUENTIAL: true
+          DEBUG: electron-builder
 
   test-mac-26:
     runs-on: macos-26

package.json

@@ -20,7 +20,7 @@
     "generate-changeset": "pnpm changeset",
     "generate-schema": "typescript-json-schema packages/app-builder-lib/tsconfig-scheme.json Configuration --out packages/app-builder-lib/scheme.json --noExtraProps --useTypeOfKeyword --strictNullChecks --required && node ./scripts/fix-schema.js",
     "generate-all": "pnpm generate-schema && pnpm prettier",
-    "ci:test": "vitest run --no-update --allowOnly",
+    "ci:test": "vitest run",
     "ci:version": "pnpm i && pnpm changelog && changeset version && node scripts/update-package-version-export.js && pnpm compile && pnpm generate-all && git add .",
     "ci:publish": "pnpm i && pnpm compile && pnpm publish -r --tag next && changeset tag",
     "docs:prebuild": "docker build -t mkdocs-dockerfile -f mkdocs-dockerfile . ",
@@ -52,6 +52,7 @@
     "eslint-config-prettier": "^9.1.0",
     "eslint-plugin-prettier": "^5.2.1",
     "fs-extra": "10.1.0",
+    "is-ci": "^4.1.0",
     "prettier": "3.3.3",
     "ts-node": "^10.9.2",
     "typedoc": "^0.26.7",

packages/app-builder-lib/src/asar/unpackDetector.ts

@@ -39,9 +39,7 @@ export function detectUnpackedDirs(fileSet: ResolvedFileSet, autoUnpackDirs: Set
       continue
     }
 
-    if (log.isDebugEnabled) {
-      log.debug({ file: stat.moduleFullFilePath, reason: "contains executable code" }, "not packed into asar archive")
-    }
+    log.debug({ file: stat.moduleFullFilePath, reason: "contains executable code" }, "not packed into asar archive")
     autoUnpackDirs.add(stat.moduleRootPath)
   }
 }

packages/app-builder-lib/src/node-module-collector/index.ts

@@ -25,7 +25,6 @@ export function getCollectorByPackageManager(pm: PM, rootDir: string, tempDirMan
     case PM.BUN:
       return new BunNodeModulesCollector(rootDir, tempDirManager)
     case PM.NPM:
-    default:
       return new NpmNodeModulesCollector(rootDir, tempDirManager)
   }
 }
@@ -72,7 +71,7 @@ async function findWorkspaceRoot(pm: PM, cwd: string): Promise<string | undefine
 
   switch (pm) {
     case PM.PNPM:
-      command = { command: "pnpm", args: ["root", "-w"] }
+      command = { command: "pnpm", args: ["--workspace-root", "exec", "pwd"] }
       break
 
     case PM.YARN_BERRY:

packages/app-builder-lib/src/node-module-collector/nodeModulesCollector.ts

@@ -1,5 +1,5 @@
 import { exists, log, retry, TmpDir } from "builder-util"
-import { spawn as spawnProcess } from "child_process"
+import * as childProcess from "child_process"
 import { CancellationToken } from "builder-util-runtime"
 import * as fs from "fs-extra"
 import { createWriteStream, readJson } from "fs-extra"
@@ -9,6 +9,7 @@ import { hoist, type HoisterResult, type HoisterTree } from "./hoist"
 import { createModuleCache, type ModuleCache } from "./moduleCache"
 import { getPackageManagerCommand, PM } from "./packageManager"
 import type { Dependency, DependencyGraph, NodeModuleInfo, PackageJson } from "./types"
+import { fileURLToPath, pathToFileURL } from "node:url"
 
 export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDepType, OptionalDepType>, OptionalDepType> {
   private nodeModules: NodeModuleInfo[] = []
@@ -17,21 +18,33 @@ export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDe
   protected pkgJsonCache: Map<string, string> = new Map()
   protected memoResolvedModules = new Map<string, Promise<string | null>>()
 
+  private importMetaResolve = new Lazy(async () => {
+    try {
+      // Node >= 16 builtin ESM-aware resolver
+      const packageName = "import-meta-resolve"
+      const imported = await import(packageName)
+      return imported.resolve
+    } catch {
+      return null
+    }
+  })
+
   // Unified cache for all file system and module operations
   protected cache: ModuleCache = createModuleCache()
 
   protected isHoisted = new Lazy<boolean>(async () => {
-    const command = getPackageManagerCommand(this.installOptions.manager)
+    const { manager } = this.installOptions
+    const command = getPackageManagerCommand(manager)
 
     const config = (await this.asyncExec(command, ["config", "list"])).stdout
     if (config == null) {
-      log.debug({ manager: this.installOptions.manager }, "unable to determine if node_modules are hoisted: no config output. falling back to hoisted mode")
+      log.debug({ manager }, "unable to determine if node_modules are hoisted: no config output. falling back to hoisted mode")
       return false
     }
     const lines = Object.fromEntries(config.split("\n").map(line => line.split("=").map(s => s.trim())))
 
     if (lines["node-linker"] === "hoisted") {
-      log.debug({ manager: this.installOptions.manager }, "node_modules are hoisted")
+      log.debug({ manager }, "node_modules are hoisted")
       return true
     }
 
@@ -57,7 +70,7 @@ export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDe
 
     await this.collectAllDependencies(tree, packageName)
 
-    const realTree: ProdDepType = await this.getTreeFromWorkspaces(tree)
+    const realTree: ProdDepType = await this.getTreeFromWorkspaces(tree, packageName)
     await this.extractProductionDependencyGraph(realTree, packageName)
 
     if (cancellationToken.cancelled) {
@@ -184,27 +197,57 @@ export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDe
    * Resolves a package to its filesystem location using Node.js module resolution.
    * Returns the directory containing the package, not the package.json path.
    */
-  protected resolvePackageDir = (packageName: string, fromDir: string): string | null => {
+  protected async resolvePackageDir(packageName: string, fromDir: string): Promise<string | null> {
     const cacheKey = `${packageName}::${fromDir}`
-
-    // Check memoization cache
     if (this.cache.requireResolve.has(cacheKey)) {
       return this.cache.requireResolve.get(cacheKey)!
     }
 
-    try {
-      // require.resolve finds the main entry point, so we look for package.json instead
-      const packageJsonPath = require.resolve(`${packageName}/package.json`, {
-        paths: [fromDir, this.rootDir],
-      })
-      const result = path.dirname(packageJsonPath)
-      this.cache.requireResolve.set(cacheKey, result)
-      return result
-    } catch (error: any) {
-      log.warn({ packageName, fromDir, error: error.message }, "could not resolve package")
+    const resolveEntry = async () => {
+      // 1) Try Node ESM resolver (handles exports reliably)
+      if (await this.importMetaResolve.value) {
+        try {
+          const url = (await this.importMetaResolve.value)(packageName, pathToFileURL(fromDir).href)
+          return url.startsWith("file://") ? fileURLToPath(url) : null
+        } catch {
+          // ignore
+        }
+      }
+
+      // 2) Fallback: require.resolve (CJS, old packages)
+      try {
+        return require.resolve(packageName, { paths: [fromDir, this.rootDir] })
+      } catch {
+        // ignore
+      }
+
+      return null
+    }
+
+    const entry = await resolveEntry()
+    if (!entry) {
       this.cache.requireResolve.set(cacheKey, null)
       return null
     }
+
+    // 3) Walk upward until you find a package.json
+    let dir = path.dirname(entry)
+    while (true) {
+      const pkgFile = path.join(dir, "package.json")
+      if (await exists(pkgFile)) {
+        this.cache.requireResolve.set(cacheKey, dir)
+        return dir
+      }
+
+      const parent = path.dirname(dir)
+      if (parent === dir) {
+        break // root reached
+      }
+      dir = parent
+    }
+
+    this.cache.requireResolve.set(cacheKey, null)
+    return null
   }
 
   protected requireMemoized(pkgPath: string): PackageJson {
@@ -244,20 +287,20 @@ export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDe
     return { name, version }
   }
 
-  protected async getTreeFromWorkspaces(tree: ProdDepType): Promise<ProdDepType> {
-    if (tree.workspaces && tree.dependencies) {
-      const packageJson = await this.appPkgJson.value
-      const dependencyName = packageJson.name
+  protected async getTreeFromWorkspaces(tree: ProdDepType, packageName: string): Promise<ProdDepType> {
+    if (!(tree.workspaces && tree.dependencies)) {
+      return tree
+    }
 
-      for (const [key, value] of Object.entries(tree.dependencies)) {
-        if (key === dependencyName) {
-          log.debug({ key, path: value.path }, "returning workspace tree for root dependency")
-          return value
-        }
+    if (tree.dependencies?.[packageName]) {
+      const { name, path, dependencies } = tree.dependencies[packageName]
+      log.debug({ name, path, dependencies: JSON.stringify(dependencies) }, "pruning root app/self package from workspace tree")
+      for (const [name, pkg] of Object.entries(dependencies ?? {})) {
+        tree.dependencies[name] = pkg
       }
+      delete tree.dependencies[packageName]
     }
-
-    return tree
+    return Promise.resolve(tree)
   }
 
   private transformToHoisterTree(obj: DependencyGraph, key: string, nodes: Map<string, HoisterTree> = new Map()): HoisterTree {
@@ -352,7 +395,7 @@ export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDe
     await new Promise<void>((resolve, reject) => {
       const outStream = createWriteStream(tempOutputFile)
 
-      const child = spawnProcess(command, args, {
+      const child = childProcess.spawn(command, args, {
         cwd,
         shell: false, // required to prevent console logs polution from shell profile loading when `true`
       })

packages/app-builder-lib/src/node-module-collector/npmNodeModulesCollector.ts

@@ -16,12 +16,11 @@ export class NpmNodeModulesCollector extends NodeModulesCollector<NpmDependency,
 
   protected async getDependenciesTree(pm: PM): Promise<NpmDependency> {
     try {
-      // force NPM collection as Yarn Berry extends this class and PnP is not supported directly
       return await super.getDependenciesTree(pm)
     } catch (error: any) {
-      log.info({ pm, parser: PM.NPM, error: error.message }, "unable to process dependency tree, falling back to using manual node_modules traversal")
+      log.info({ pm: this.installOptions.manager, parser: PM.NPM, error: error.message }, "unable to process dependency tree, falling back to using manual node_modules traversal")
     }
-    // node_modules linker fallback. (Slower due to system ops, so we only use it as a fallback)
+    // node_modules linker fallback. (Slower due to system ops, so we only use it as a fallback) [e.g. corepack env will not allow npm CLI to extract tree]
     return this.buildNodeModulesTreeManually(this.rootDir)
   }
 
@@ -110,13 +109,21 @@ export class NpmNodeModulesCollector extends NodeModulesCollector<NpmDependency,
       for (const [depName, depVersion] of Object.entries(allProdDepNames)) {
         try {
           // Resolve the dependency using Node.js module resolution from this package's directory
-          const depPath = this.resolvePackageDir(depName, packageDir)
+          const depPath = await this.resolvePackageDir(depName, packageDir)
 
           if (!depPath) {
             log.warn({ package: pkg.name, dependency: depName, version: depVersion }, "dependency not found, skipping")
             continue
           }
 
+          const resolvedDepPath = await this.resolvePath(depPath)
+
+          // Skip if this dependency resolves to the base directory or any parent we're already processing
+          if (resolvedDepPath === resolvedPackageDir || resolvedDepPath === (await this.resolvePath(baseDir))) {
+            log.debug({ package: pkg.name, dependency: depName, resolvedPath: resolvedDepPath }, "skipping self-referential dependency")
+            continue
+          }
+
           log.debug({ package: pkg.name, dependency: depName, resolvedPath: depPath }, "processing production dependency")
 
           // Recursively build the dependency tree for this dependency

packages/app-builder-lib/src/node-module-collector/packageManager.ts

@@ -4,11 +4,11 @@ import * as path from "path"
 import * as which from "which"
 
 export enum PM {
-  NPM = "npm",
-  YARN = "yarn",
   PNPM = "pnpm",
+  YARN = "yarn",
   YARN_BERRY = "yarn-berry",
   BUN = "bun",
+  NPM = "npm",
 }
 
 // Cache for resolved paths

packages/app-builder-lib/src/node-module-collector/pnpmNodeModulesCollector.ts

@@ -1,4 +1,4 @@
-import { log } from "builder-util"
+import { isEmptyOrSpaces, log } from "builder-util"
 import * as path from "path"
 import { NodeModulesCollector } from "./nodeModulesCollector"
 import { PM } from "./packageManager"
@@ -19,8 +19,14 @@ export class PnpmNodeModulesCollector extends NodeModulesCollector<PnpmDependenc
       return
     }
 
-    const getProductionDependencies = async (tree: PnpmDependency): Promise<{ prodDeps: Record<string, string>; optionalDependencies: Record<string, string> } | null> => {
-      const p = path.normalize(this.resolvePackageDir(tree.name, tree.path) ?? (await this.resolvePath(tree.path)))
+    const getProductionDependencies = async (depTree: PnpmDependency): Promise<{ prodDeps: Record<string, string>; optionalDependencies: Record<string, string> } | null> => {
+      const packageName = depTree.name || depTree.from
+      if (isEmptyOrSpaces(packageName)) {
+        log.error(depTree, `Cannot determine production dependencies for package with empty name`)
+        throw new Error(`Cannot compute production dependencies for package with empty name: ${packageName}`)
+      }
+
+      const p = path.normalize((await this.resolvePackageDir(packageName, depTree.path)) ?? (await this.resolvePath(depTree.path)))
       const pkgJsonPath = path.join(p, "package.json")
 
       let packageJson: PackageJson
@@ -33,7 +39,8 @@ export class PnpmNodeModulesCollector extends NodeModulesCollector<PnpmDependenc
       return { prodDeps: { ...packageJson.dependencies, ...packageJson.optionalDependencies }, optionalDependencies: { ...packageJson.optionalDependencies } }
     }
 
-    const json = tree.name === dependencyId ? null : await getProductionDependencies(tree)
+    const packageName = tree.name || tree.from
+    const json = packageName === dependencyId ? null : await getProductionDependencies(tree)
     const prodDependencies = json?.prodDeps ?? { ...(tree.dependencies || {}), ...(tree.optionalDependencies || {}) }
     if (prodDependencies == null) {
       this.productionGraph[dependencyId] = { dependencies: [] }

packages/app-builder-lib/src/node-module-collector/yarnBerryNodeModulesCollector.ts

@@ -47,6 +47,7 @@ export class YarnBerryNodeModulesCollector extends NpmNodeModulesCollector {
     const output = await this.asyncExec("yarn", ["config", "--json"], rootDir)
 
     if (!output.stdout) {
+      log.debug(null, "Yarn config returned no output, assuming default Yarn v1 behavior (hoisted, non-PnP)")
       return {
         yarnVersion,
         nodeLinker,