fix: double check "hoisted" mode for each node module, verify through filesystem check instead of require/import-meta-resolve methods (#9401)

Author: mmaietta

.changeset/yellow-loops-wink.md

@@ -0,0 +1,5 @@
+---
+"app-builder-lib": patch
+---
+
+fix: double check yarn hoisted mode for each node module - yarn classic doesn't perma-hoist

packages/app-builder-lib/src/asar/asar.ts

@@ -48,6 +48,9 @@ export class AsarFilesystem {
     let node = this.header
     for (const dir of p.split(path.sep)) {
       if (dir !== ".") {
+        if (node == null) {
+          throw new Error(`Cannot find node for path: ${p} (node is null at ${dir})`)
+        }
         let child = node.files![dir]
         if (child == null) {
           if (!isCreate) {
@@ -70,6 +73,9 @@ export class AsarFilesystem {
 
     const name = path.basename(p)
     const dirNode = this.searchNodeFromDirectory(path.dirname(p), true)!
+    if (dirNode == null) {
+      throw new Error(`Cannot find node for path: ${p} (node is null at ${path.dirname(p)})`)
+    }
     if (dirNode.files == null) {
       dirNode.files = this.newNode()
     }
@@ -115,6 +121,9 @@ export class AsarFilesystem {
 
   getNode(p: string): Node | null {
     const node = this.searchNodeFromDirectory(path.dirname(p), false)!
+    if (node == null) {
+      throw new Error(`Cannot find node for path: ${p} (node is null at ${path.dirname(p)})`)
+    }
     return node.files![path.basename(p)]
   }
 

packages/app-builder-lib/src/node-module-collector/moduleCache.ts

@@ -14,7 +14,7 @@ export type ModuleCache = {
   /** Cache for lstat results */
   lstat: Map<string, fs.Stats>
   /** Cache for require.resolve results (key: "packageName::fromDir") */
-  requireResolve: Map<string, string | null>
+  requireResolve: Map<string, { entry: string; packageDir: string } | null>
 }
 
 /**

packages/app-builder-lib/src/node-module-collector/nodeModulesCollector.ts

@@ -1,4 +1,4 @@
-import { exists, log, retry, TmpDir } from "builder-util"
+import { exists, isEmptyOrSpaces, log, retry, TmpDir } from "builder-util"
 import * as childProcess from "child_process"
 import { CancellationToken } from "builder-util-runtime"
 import * as fs from "fs-extra"
@@ -9,35 +9,21 @@ import { hoist, type HoisterResult, type HoisterTree } from "./hoist"
 import { createModuleCache, type ModuleCache } from "./moduleCache"
 import { getPackageManagerCommand, PM } from "./packageManager"
 import type { Dependency, DependencyGraph, NodeModuleInfo, PackageJson } from "./types"
-import { fileURLToPath, pathToFileURL } from "node:url"
 
 export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDepType, OptionalDepType>, OptionalDepType> {
-  private nodeModules: NodeModuleInfo[] = []
-  protected allDependencies: Map<string, ProdDepType> = new Map()
-  protected productionGraph: DependencyGraph = {}
-  protected pkgJsonCache: Map<string, string> = new Map()
-  protected memoResolvedModules = new Map<string, Promise<string | null>>()
-
-  private importMetaResolve = new Lazy(async () => {
-    try {
-      // Node >= 16 builtin ESM-aware resolver
-      const packageName = "import-meta-resolve"
-      const imported = await import(packageName)
-      return imported.resolve
-    } catch {
-      return null
-    }
-  })
+  private readonly nodeModules: NodeModuleInfo[] = []
+  protected readonly allDependencies: Map<string, ProdDepType> = new Map()
+  protected readonly productionGraph: DependencyGraph = {}
 
   // Unified cache for all file system and module operations
-  protected cache: ModuleCache = createModuleCache()
+  private readonly cache: ModuleCache = createModuleCache()
 
-  protected isHoisted = new Lazy<boolean>(async () => {
+  protected readonly isHoisted = new Lazy<boolean>(async () => {
     const { manager } = this.installOptions
     const command = getPackageManagerCommand(manager)
 
-    const config = (await this.asyncExec(command, ["config", "list"])).stdout
-    if (config == null) {
+    const config = (await this.asyncExec(command, ["config", "list"])).stdout?.trim()
+    if (isEmptyOrSpaces(config)) {
       log.debug({ manager }, "unable to determine if node_modules are hoisted: no config output. falling back to hoisted mode")
       return false
     }
@@ -69,7 +55,6 @@ export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDe
     }
 
     await this.collectAllDependencies(tree, packageName)
-
     const realTree: ProdDepType = await this.getTreeFromWorkspaces(tree, packageName)
     await this.extractProductionDependencyGraph(realTree, packageName)
 
@@ -80,10 +65,8 @@ export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDe
     const hoisterResult: HoisterResult = hoist(this.transformToHoisterTree(this.productionGraph, packageName), {
       check: log.isDebugEnabled,
     })
-
     await this._getNodeModules(hoisterResult.dependencies, this.nodeModules)
     log.debug({ packageName, depCount: this.nodeModules.length }, "node modules collection complete")
-
     return this.nodeModules
   }
 
@@ -210,60 +193,52 @@ export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDe
   }
 
   /**
-   * Resolves a package to its filesystem location using Node.js module resolution.
-   * Returns the directory containing the package, not the package.json path.
+   * Resolve a package directory purely from the filesystem.
+   * Does NOT attempt to load the module or resolve an "exports" entrypoint.
+   * Good for Yarn 4 because a package may not be resolvable as a module,
+   * but still exists on disk.
    */
-  protected async resolvePackageDir(packageName: string, fromDir: string): Promise<string | null> {
+  protected async resolvePackage(packageName: string, fromDir: string): Promise<{ entry: string; packageDir: string } | null> {
     const cacheKey = `${packageName}::${fromDir}`
     if (this.cache.requireResolve.has(cacheKey)) {
       return this.cache.requireResolve.get(cacheKey)!
     }
 
-    const resolveEntry = async () => {
-      // 1) Try Node ESM resolver (handles exports reliably)
-      if (await this.importMetaResolve.value) {
-        try {
-          const url = (await this.importMetaResolve.value)(packageName, pathToFileURL(fromDir).href)
-          return url.startsWith("file://") ? fileURLToPath(url) : null
-        } catch (error: any) {
-          log.debug({ error: error.message }, "import-meta-resolve failed")
-          // ignore
-        }
-      }
-
-      // 2) Fallback: require.resolve (CJS, old packages)
-      try {
-        return require.resolve(packageName, { paths: [fromDir, this.rootDir] })
-      } catch (error: any) {
-        log.debug({ error: error.message }, "require.resolve failed")
-        // ignore
-      }
-
-      return null
+    // 1. NESTED under fromDir/node_modules/<name>
+    let candidate = path.join(fromDir, "node_modules", packageName)
+    let pkgJson = path.join(candidate, "package.json")
+    if (await this.existsMemoized(pkgJson)) {
+      this.cache.requireResolve.set(cacheKey, { entry: pkgJson, packageDir: candidate })
+      return { entry: pkgJson, packageDir: candidate }
     }
 
-    const entry = await resolveEntry()
-    if (!entry) {
-      this.cache.requireResolve.set(cacheKey, null)
-      return null
+    // 2. HOISTED under rootDir/node_modules/<name>
+    candidate = path.join(this.rootDir, "node_modules", packageName)
+    pkgJson = path.join(candidate, "package.json")
+    if (await this.existsMemoized(pkgJson)) {
+      this.cache.requireResolve.set(cacheKey, { entry: pkgJson, packageDir: candidate })
+      return { entry: pkgJson, packageDir: candidate }
     }
 
-    // 3) Walk upward until you find a package.json
-    let dir = path.dirname(entry)
+    // 3. FALLBACK: try parent directories BFS (classic Node-style search)
+    let current = fromDir
     while (true) {
-      const pkgFile = path.join(dir, "package.json")
-      if (await this.existsMemoized(pkgFile)) {
-        this.cache.requireResolve.set(cacheKey, dir)
-        return dir
+      const nm = path.join(current, "node_modules", packageName)
+      const pkg = path.join(nm, "package.json")
+
+      if (await this.existsMemoized(pkg)) {
+        this.cache.requireResolve.set(cacheKey, { entry: pkg, packageDir: nm })
+        return { entry: pkg, packageDir: nm }
       }
 
-      const parent = path.dirname(dir)
-      if (parent === dir) {
-        break // root reached
+      const parent = path.dirname(current)
+      if (parent === current) {
+        break
       }
-      dir = parent
+      current = parent
     }
 
+    // 4. LAST RESORT: DO NOT throw — just return null
     this.cache.requireResolve.set(cacheKey, null)
     return null
   }
@@ -298,9 +273,10 @@ export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDe
 
     if (tree.dependencies?.[packageName]) {
       const { name, path, dependencies } = tree.dependencies[packageName]
-      log.debug({ name, path, dependencies: JSON.stringify(dependencies) }, "pruning root app/self reference from workspace tree")
+      log.debug({ name, path, dependencies: JSON.stringify(dependencies) }, "pruning root app/self reference from workspace tree, merging dependencies uptree")
       for (const [name, pkg] of Object.entries(dependencies ?? {})) {
         tree.dependencies[name] = pkg
+        this.allDependencies.set(this.packageVersionString(pkg), pkg)
       }
       delete tree.dependencies[packageName]
     }
@@ -396,6 +372,8 @@ export abstract class NodeModulesCollector<ProdDepType extends Dependency<ProdDe
       args = ["/c", tempBatFile, ...args]
     }
 
+    log.debug({ command, args, cwd, tempOutputFile }, "spawning node module collector process")
+
     await new Promise<void>((resolve, reject) => {
       const outStream = createWriteStream(tempOutputFile)
 

packages/app-builder-lib/src/node-module-collector/npmNodeModulesCollector.ts

@@ -84,7 +84,7 @@ export class NpmNodeModulesCollector extends NodeModulesCollector<NpmDependency,
       }
 
       // Read package.json using memoized require for consistency with Node.js module system
-      const pkg: PackageJson = this.requireMemoized(pkgPath)
+      const pkg: PackageJson = await this.readJsonMemoized(pkgPath)
       const resolvedPackageDir = await this.resolvePath(packageDir)
 
       // Use resolved path as the unique identifier to prevent circular dependencies
@@ -109,25 +109,23 @@ export class NpmNodeModulesCollector extends NodeModulesCollector<NpmDependency,
       for (const [depName, depVersion] of Object.entries(allProdDepNames)) {
         try {
           // Resolve the dependency using Node.js module resolution from this package's directory
-          const depPath = await this.resolvePackageDir(depName, packageDir)
+          const resolvedPackage = await this.resolvePackage(depName, packageDir)
 
-          if (!depPath) {
-            log.warn({ package: pkg.name, dependency: depName, version: depVersion }, "dependency not found, skipping")
-            continue
+          if (!resolvedPackage) {
+            log.warn({ package: pkg.name, dependency: depName, version: depVersion }, "dependency not found")
           }
 
-          const resolvedDepPath = await this.resolvePath(depPath)
-
+          const resolvedDepPath = await this.resolvePath(resolvedPackage!.packageDir)
           // Skip if this dependency resolves to the base directory or any parent we're already processing
           if (resolvedDepPath === resolvedPackageDir || resolvedDepPath === (await this.resolvePath(baseDir))) {
             log.debug({ package: pkg.name, dependency: depName, resolvedPath: resolvedDepPath }, "skipping self-referential dependency")
             continue
           }
 
-          log.debug({ package: pkg.name, dependency: depName, resolvedPath: depPath }, "processing production dependency")
+          log.debug({ package: pkg.name, dependency: depName, ...resolvedPackage, resolvedDepPath }, "processing production dependency")
 
           // Recursively build the dependency tree for this dependency
-          prodDeps[depName] = await buildFromPackage(depPath)
+          prodDeps[depName] = await buildFromPackage(resolvedDepPath)
         } catch (error: any) {
           log.warn({ package: pkg.name, dependency: depName, error: error.message }, "failed to process dependency, skipping")
         }

packages/app-builder-lib/src/node-module-collector/packageManager.ts

@@ -58,23 +58,23 @@ export async function detectPackageManager(searchPaths: string[]): Promise<{ pm:
       const [pm, version] = packageManager.split("@")
       if (Object.values(PM).includes(pm as PM)) {
         const resolvedPackageManager = await resolveIfYarn(pm as PM, version, dir)
-        log.debug({ resolvedPackageManager, packageManager, cwd: dir }, "packageManager field detected in package.json")
+        log.info({ resolvedPackageManager, packageManager, cwd: dir }, "packageManager field detected in package.json")
         return { pm: resolvedPackageManager, corepackConfig: packageManager, resolvedDirectory: dir }
       }
     }
 
     pm = await detectPackageManagerByFile(dir)
     if (pm) {
       const resolvedPackageManager = await resolveIfYarn(pm, "", dir)
-      log.debug({ resolvedPackageManager, cwd: dir }, "packageManager detected by file")
+      log.info({ resolvedPackageManager, cwd: dir }, "packageManager detected by file")
       return { pm: resolvedPackageManager, resolvedDirectory: dir, corepackConfig: undefined }
     }
   }
 
   pm = detectPackageManagerByEnv() || PM.NPM
   const cwd = process.env.npm_package_json ? path.dirname(process.env.npm_package_json) : (process.env.INIT_CWD ?? process.cwd())
   const resolvedPackageManager = await resolveIfYarn(pm, "", cwd)
-  log.debug({ resolvedPackageManager, detected: cwd }, "packageManager not detected by file, falling back to environment detection")
+  log.info({ resolvedPackageManager, detected: cwd }, "packageManager not detected by file, falling back to environment detection")
   return { pm: resolvedPackageManager, resolvedDirectory: undefined, corepackConfig: undefined }
 }
 

packages/app-builder-lib/src/node-module-collector/pnpmNodeModulesCollector.ts

@@ -36,19 +36,20 @@ export class PnpmNodeModulesCollector extends NodeModulesCollector<PnpmDependenc
       throw new Error(`Cannot compute production dependencies for package with empty name: ${packageName}`)
     }
 
-    const actualPath = await this.resolveActualPath(depTree)
-    const resolvedLocalPath = await this.resolvePath(actualPath)
-    const p = path.normalize(resolvedLocalPath)
-    const pkgJsonPath = path.join(p, "package.json")
+    const pkgJsonPath = await this.resolvePackage(packageName, depTree.path)
 
+    if (pkgJsonPath == null) {
+      log.warn({ packageName, path: depTree.path, version: depTree.version }, `Cannot find package.json for dependency`)
+      return { path: depTree.path, prodDeps: {}, optionalDependencies: {} }
+    }
     let packageJson: PackageJson
     try {
-      packageJson = this.requireMemoized(pkgJsonPath)
+      packageJson = await this.readJsonMemoized(pkgJsonPath.entry)
     } catch (error: any) {
-      log.warn(null, `Failed to read package.json for ${p}: ${error.message}`)
-      return { path: p, prodDeps: {}, optionalDependencies: {} }
+      log.warn(pkgJsonPath, `Failed to read package.json: ${error.message}`)
+      return { path: pkgJsonPath.packageDir, prodDeps: {}, optionalDependencies: {} }
     }
-    return { path: p, prodDeps: { ...packageJson.dependencies, ...packageJson.optionalDependencies }, optionalDependencies: { ...packageJson.optionalDependencies } }
+    return { path: pkgJsonPath.packageDir, prodDeps: { ...packageJson.dependencies, ...packageJson.optionalDependencies }, optionalDependencies: { ...packageJson.optionalDependencies } }
   }
 
   protected async extractProductionDependencyGraph(tree: PnpmDependency, dependencyId: string) {