Support bundled dependencies in resolver plugin (#4903)

* [lookup-by-path] Return linked list of matches

* [workspace-resolve-plugin] Handle hierarchical node_modules

* [rush-resolve-plugin] Support "bundledDependencies"

* Apply suggestions from code review

Reformat change logs, add comments about numeric values.

Co-authored-by: Ian Clanton-Thuon <[email protected]>

---------

Co-authored-by: David Michon <[email protected]>
Co-authored-by: Ian Clanton-Thuon <[email protected]>

Author: dmichon-msft

common/changes/@microsoft/rush/bundled-dependencies_2024-08-24-01-03.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Support `bundledDependencies` in rush-resolver-cache-plugin.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/changes/@rushstack/lookup-by-path/bundled-dependencies_2024-08-26-18-08.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/lookup-by-path",
+      "comment": "Return a linked list of matches in `findLongestPrefixMatch` in the event that multiple prefixes match. The head of the list is the most specific match.",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@rushstack/lookup-by-path"
+}

common/changes/@rushstack/webpack-workspace-resolve-plugin/bundled-dependencies_2024-08-24-01-03.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@rushstack/webpack-workspace-resolve-plugin",
+      "comment": "Support hierarchical `node_modules` folders.",
+      "type": "minor"
+    }
+  ],
+  "packageName": "@rushstack/webpack-workspace-resolve-plugin"
+}

common/reviews/api/lookup-by-path.api.md

@@ -7,6 +7,7 @@
 // @beta
 export interface IPrefixMatch<TItem> {
     index: number;
+    lastMatch?: IPrefixMatch<TItem>;
     value: TItem;
 }
 

common/reviews/api/webpack-workspace-resolve-plugin.api.md

@@ -25,7 +25,7 @@ export interface IResolverCacheFile {
 
 // @beta
 export interface ISerializedResolveContext {
-    deps: Record<string, number>;
+    deps?: Record<string, number>;
     dirInfoFiles?: string[];
     name: string;
     root: string;
@@ -46,7 +46,7 @@ export interface IWorkspaceResolvePluginOptions {
 // @beta
 export class WorkspaceLayoutCache {
     constructor(options: IWorkspaceLayoutCacheOptions);
-    readonly contextForPackage: WeakMap<object, IResolveContext>;
+    readonly contextForPackage: WeakMap<object, IPrefixMatch<IResolveContext>>;
     readonly contextLookup: LookupByPath<IResolveContext>;
     // (undocumented)
     readonly normalizeToPlatform: IPathNormalizationFunction;

libraries/lookup-by-path/src/LookupByPath.test.ts

@@ -133,7 +133,11 @@ describe(LookupByPath.prototype.findLongestPrefixMatch.name, () => {
       ['foo/bar', 4]
     ]);
 
-    expect(tree.findLongestPrefixMatch('foo/bar')).toEqual({ value: 4, index: 7 });
+    expect(tree.findLongestPrefixMatch('foo/bar')).toEqual({
+      value: 4,
+      index: 7,
+      lastMatch: { value: 1, index: 3 }
+    });
     expect(tree.findLongestPrefixMatch('barbar/baz')).toEqual({ value: 2, index: 6 });
     expect(tree.findLongestPrefixMatch('baz/foo')).toEqual({ value: 3, index: 3 });
     expect(tree.findLongestPrefixMatch('foo/foo')).toEqual({ value: 1, index: 3 });

libraries/lookup-by-path/src/LookupByPath.ts

@@ -40,6 +40,10 @@ export interface IPrefixMatch<TItem> {
    * The index of the first character after the matched prefix
    */
   index: number;
+  /**
+   * The last match found (with a shorter prefix), if any
+   */
+  lastMatch?: IPrefixMatch<TItem>;
 }
 
 /**
@@ -255,7 +259,8 @@ export class LookupByPath<TItem> {
     let best: IPrefixMatch<TItem> | undefined = node.value
       ? {
           value: node.value,
-          index: 0
+          index: 0,
+          lastMatch: undefined
         }
       : undefined;
     // Trivial cases
@@ -269,7 +274,8 @@ export class LookupByPath<TItem> {
         if (node.value !== undefined) {
           best = {
             value: node.value,
-            index
+            index,
+            lastMatch: best
           };
         }
         if (!node.children) {

libraries/rush-lib/src/logic/pnpm/PnpmShrinkwrapFile.ts

@@ -59,6 +59,8 @@ export interface IPnpmShrinkwrapDependencyYaml {
     /** The name of the tarball, if this was from a TGZ file */
     tarball?: string;
   };
+  /** The list of bundled dependencies in this package */
+  bundledDependencies?: ReadonlyArray<string>;
   /** The list of dependencies and the resolved version */
   dependencies?: Record<string, IPnpmVersionSpecifier>;
   /** The list of optional dependencies and the resolved version */

rush-plugins/rush-resolver-cache-plugin/src/afterInstallAsync.ts

@@ -117,8 +117,12 @@ export async function afterInstallAsync(
 
         const filteredFiles: string[] = Object.keys(files).filter((file) => file.endsWith('/package.json'));
         if (filteredFiles.length > 0) {
-          // eslint-disable-next-line require-atomic-updates
-          context.files = filteredFiles.map((x) => x.slice(0, -13));
+          const nestedPackageDirs: string[] = filteredFiles.map((x) => x.slice(0, /* -'/package.json'.length */ -13));
+
+          if (nestedPackageDirs.length > 0) {
+            // eslint-disable-next-line require-atomic-updates
+            context.nestedPackageDirs = nestedPackageDirs;
+          }
         }
       } catch (error) {
         if (!context.optional) {

rush-plugins/rush-resolver-cache-plugin/src/computeResolverCacheFromLockfileAsync.ts

@@ -45,6 +45,59 @@ function isPackageCompatible(
   return true;
 }
 
+function extractBundledDependencies(
+  contexts: Map<string, IResolverContext>,
+  context: IResolverContext
+): void {
+  const { nestedPackageDirs } = context;
+  if (!nestedPackageDirs) {
+    return;
+  }
+
+  for (let i: number = nestedPackageDirs.length - 1; i >= 0; i--) {
+    const nestedDir: string = nestedPackageDirs[i];
+    if (!nestedDir.startsWith('node_modules/')) {
+      continue;
+    }
+
+    const isScoped: boolean = nestedDir.charAt(/* 'node_modules/'.length */ 13) === '@';
+    let index: number = nestedDir.indexOf('/', 13);
+    if (isScoped) {
+      index = nestedDir.indexOf('/', index + 1);
+    }
+
+    const name: string = index === -1 ? nestedDir.slice(13) : nestedDir.slice(13, index);
+    if (name.startsWith('.')) {
+      continue;
+    }
+
+    // Remove this nested package from the list
+    nestedPackageDirs.splice(i, 1);
+
+    const remainder: string = index === -1 ? '' : nestedDir.slice(index + 1);
+    const nestedRoot: string = `${context.descriptionFileRoot}/node_modules/${name}`;
+    let nestedContext: IResolverContext | undefined = contexts.get(nestedRoot);
+    if (!nestedContext) {
+      nestedContext = {
+        descriptionFileRoot: nestedRoot,
+        descriptionFileHash: undefined,
+        isProject: false,
+        name,
+        deps: new Map(),
+        ordinal: -1
+      };
+      contexts.set(nestedRoot, nestedContext);
+    }
+
+    context.deps.set(name, nestedRoot);
+
+    if (remainder) {
+      nestedContext.nestedPackageDirs ??= [];
+      nestedContext.nestedPackageDirs.push(remainder);
+    }
+  }
+}
+
 /**
  * Options for computing the resolver cache from a lockfile.
  */
@@ -130,7 +183,7 @@ export async function computeResolverCacheFromLockfileAsync(
       isProject: false,
       name,
       deps: new Map(),
-      ordinal: contexts.size,
+      ordinal: -1,
       optional: pack.optional
     };
 
@@ -148,6 +201,12 @@ export async function computeResolverCacheFromLockfileAsync(
     await afterExternalPackagesAsync(contexts, missingOptionalDependencies);
   }
 
+  for (const context of contexts.values()) {
+    if (context.nestedPackageDirs) {
+      extractBundledDependencies(contexts, context);
+    }
+  }
+
   // Add the data for workspace projects
   for (const [importerPath, importer] of lockfile.importers) {
     // Ignore the root project. This plugin assumes you don't have one.
@@ -167,7 +226,7 @@ export async function computeResolverCacheFromLockfileAsync(
       name: project.packageJson.name,
       isProject: true,
       deps: new Map(),
-      ordinal: contexts.size
+      ordinal: -1
     };
 
     contexts.set(project.projectFolder, context);
@@ -183,6 +242,11 @@ export async function computeResolverCacheFromLockfileAsync(
     }
   }
 
+  let ordinal: number = 0;
+  for (const context of contexts.values()) {
+    context.ordinal = ordinal++;
+  }
+
   // Convert the intermediate representation to the final cache file
   const serializedContexts: ISerializedResolveContext[] = Array.from(
     contexts,