Add file-based transfer APIs to build cache provider interface and all cache plugins

common/changes/@microsoft/rush/copilot-stream-cache-entry-for-http-plugin_2026-04-05-03-56.json

@@ -0,0 +1,11 @@
+{
+  "changes": [
+    {
+      "comment": "Add optional file-based transfer APIs (`tryDownloadCacheEntryToFileAsync`, `tryUploadCacheEntryFromFileAsync`) to `ICloudBuildCacheProvider`, allowing cache plugins to transfer cache entries directly to and from files on disk without buffering entire contents in memory. Implement in `@rushstack/rush-http-build-cache-plugin`, `@rushstack/rush-amazon-s3-build-cache-plugin`, and `@rushstack/rush-azure-storage-build-cache-plugin`. Gated behind the `useDirectFileTransfersForBuildCache` experiment.",
+      "type": "none",
+      "packageName": "@microsoft/rush"
+    }
+  ],
+  "packageName": "@microsoft/rush",
+  "email": "198982749+Copilot@users.noreply.github.com"
+}

common/reviews/api/rush-amazon-s3-build-cache-plugin.api.md

@@ -15,6 +15,7 @@ import { WebClient } from '@rushstack/rush-sdk/lib/utilities/WebClient';
 // @public
 export class AmazonS3Client {
     constructor(credentials: IAmazonS3Credentials | undefined, options: IAmazonS3BuildCacheProviderOptionsAdvanced, webClient: WebClient, terminal: ITerminal);
+    downloadObjectToFileAsync(objectName: string, localFilePath: string): Promise<boolean>;
     // (undocumented)
     getObjectAsync(objectName: string): Promise<Buffer | undefined>;
     // (undocumented)
@@ -25,6 +26,7 @@ export class AmazonS3Client {
     static tryDeserializeCredentials(credentialString: string | undefined): IAmazonS3Credentials | undefined;
     // (undocumented)
     uploadObjectAsync(objectName: string, objectBuffer: Buffer): Promise<void>;
+    uploadObjectFromFileAsync(objectName: string, localFilePath: string): Promise<void>;
     // (undocumented)
     static UriEncode(input: string): string;
 }

common/reviews/api/rush-lib.api.md

@@ -312,7 +312,7 @@ export class ExperimentsConfiguration {
 // @beta
 export class FileSystemBuildCacheProvider {
     constructor(options: IFileSystemBuildCacheProviderOptions);
-    getCacheEntryPath(cacheId: string): string;
+    readonly getCacheEntryPath: (cacheId: string) => string;
     tryGetCacheEntryPathByIdAsync(terminal: ITerminal, cacheId: string): Promise<string | undefined>;
     trySetCacheEntryBufferAsync(terminal: ITerminal, cacheId: string, entryBuffer: Buffer): Promise<string>;
 }
@@ -345,10 +345,12 @@ export interface ICloudBuildCacheProvider {
     deleteCachedCredentialsAsync(terminal: ITerminal): Promise<void>;
     // (undocumented)
     readonly isCacheWriteAllowed: boolean;
+    tryDownloadCacheEntryToFileAsync?(terminal: ITerminal, cacheId: string, localFilePath: string): Promise<boolean>;
     // (undocumented)
     tryGetCacheEntryBufferByIdAsync(terminal: ITerminal, cacheId: string): Promise<Buffer | undefined>;
     // (undocumented)
     trySetCacheEntryBufferAsync(terminal: ITerminal, cacheId: string, entryBuffer: Buffer): Promise<boolean>;
+    tryUploadCacheEntryFromFileAsync?(terminal: ITerminal, cacheId: string, localFilePath: string): Promise<boolean>;
     // (undocumented)
     updateCachedCredentialAsync(terminal: ITerminal, credential: string): Promise<void>;
     // (undocumented)
@@ -481,6 +483,7 @@ export interface IExperimentsJson {
     omitImportersFromPreventManualShrinkwrapChanges?: boolean;
     printEventHooksOutputToConsole?: boolean;
     rushAlerts?: boolean;
+    useDirectFileTransfersForBuildCache?: boolean;
     useIPCScriptsInWatchMode?: boolean;
     usePnpmFrozenLockfileForRushInstall?: boolean;
     usePnpmLockfileOnlyThenFrozenLockfileForRushUpdate?: boolean;
@@ -594,6 +597,7 @@ export interface _IOperationBuildCacheOptions {
     buildCacheConfiguration: BuildCacheConfiguration;
     excludeAppleDoubleFiles: boolean;
     terminal: ITerminal;
+    useDirectFileTransfersForBuildCache: boolean;
 }
 
 // @alpha

libraries/rush-lib/src/api/ExperimentsConfiguration.ts

@@ -136,6 +136,14 @@ export interface IExperimentsJson {
    * be included in the shared build cache.
    */
   omitAppleDoubleFilesFromBuildCache?: boolean;
+
+  /**
+   * If true, the build cache will use file-based APIs to transfer cache entries to and from cloud
+   * storage. This avoids loading the entire cache entry into memory, which can prevent out-of-memory
+   * errors for large build outputs. The cloud cache provider plugin must implement the optional
+   * file-based methods for this to take effect; otherwise it falls back to the buffer-based approach.
+   */
+  useDirectFileTransfersForBuildCache?: boolean;
 }
 
 const _EXPERIMENTS_JSON_SCHEMA: JsonSchema = JsonSchema.fromLoadedObject(schemaJson);

libraries/rush-lib/src/cli/scriptActions/PhasedScriptAction.ts

@@ -350,6 +350,11 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> i
   public async runAsync(): Promise<void> {
     const stopwatch: Stopwatch = Stopwatch.start();
 
+    const {
+      defaultSubspace,
+      subspacesFeatureEnabled,
+      pnpmOptions: { useWorkspaces }
+    } = this.rushConfiguration;
     if (this._alwaysInstall || this._installParameter?.value) {
       await measureAsyncFn(`${PERF_PREFIX}:install`, async () => {
         const { doBasicInstallAsync } = await import(
@@ -373,7 +378,7 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> i
           afterInstallAsync: (subspace: Subspace) =>
             this.rushSession.hooks.afterInstall.promise(this, subspace, variant),
           // Eventually we may want to allow a subspace to be selected here
-          subspace: this.rushConfiguration.defaultSubspace
+          subspace: defaultSubspace
         });
       });
     }
@@ -382,14 +387,12 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> i
       await measureAsyncFn(`${PERF_PREFIX}:checkInstallFlag`, async () => {
         // TODO: Replace with last-install.flag when "rush link" and "rush unlink" are removed
         const lastLinkFlag: FlagFile = new FlagFile(
-          this.rushConfiguration.defaultSubspace.getSubspaceTempFolderPath(),
+          defaultSubspace.getSubspaceTempFolderPath(),
           RushConstants.lastLinkFlagFilename,
           {}
         );
         // Only check for a valid link flag when subspaces is not enabled
-        if (!(await lastLinkFlag.isValidAsync()) && !this.rushConfiguration.subspacesFeatureEnabled) {
-          const useWorkspaces: boolean =
-            this.rushConfiguration.pnpmOptions && this.rushConfiguration.pnpmOptions.useWorkspaces;
+        if (!(await lastLinkFlag.isValidAsync()) && !subspacesFeatureEnabled) {
           if (useWorkspaces) {
             throw new Error('Link flag invalid.\nDid you run "rush install" or "rush update"?');
           } else {
@@ -513,18 +516,27 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> i
           ).IPCOperationRunnerPlugin().apply(this.hooks);
         }
 
+        const {
+          experimentsConfiguration: {
+            configuration: {
+              buildCacheWithAllowWarningsInSuccessfulBuild = false,
+              buildSkipWithAllowWarningsInSuccessfulBuild,
+              omitAppleDoubleFilesFromBuildCache: excludeAppleDoubleFiles = false,
+              useDirectFileTransfersForBuildCache = false,
+              usePnpmSyncForInjectedDependencies
+            }
+          },
+          isPnpm
+        } = this.rushConfiguration;
         if (buildCacheConfiguration?.buildCacheEnabled) {
           terminal.writeVerboseLine(`Incremental strategy: cache restoration`);
           new CacheableOperationPlugin({
-            allowWarningsInSuccessfulBuild:
-              !!this.rushConfiguration.experimentsConfiguration.configuration
-                .buildCacheWithAllowWarningsInSuccessfulBuild,
+            allowWarningsInSuccessfulBuild: buildCacheWithAllowWarningsInSuccessfulBuild,
             buildCacheConfiguration,
             cobuildConfiguration,
             terminal,
-            excludeAppleDoubleFiles:
-              !!this.rushConfiguration.experimentsConfiguration.configuration
-                .omitAppleDoubleFilesFromBuildCache
+            excludeAppleDoubleFiles,
+            useDirectFileTransfersForBuildCache
           }).apply(this.hooks);
 
           if (this._debugBuildCacheIdsParameter.value) {
@@ -534,9 +546,7 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> i
           terminal.writeVerboseLine(`Incremental strategy: output preservation`);
           // Explicitly disabling the build cache also disables legacy skip detection.
           new LegacySkipPlugin({
-            allowWarningsInSuccessfulBuild:
-              this.rushConfiguration.experimentsConfiguration.configuration
-                .buildSkipWithAllowWarningsInSuccessfulBuild,
+            allowWarningsInSuccessfulBuild: buildSkipWithAllowWarningsInSuccessfulBuild,
             terminal,
             changedProjectsOnly,
             isIncrementalBuildAllowed: this._isIncrementalBuildAllowed
@@ -551,12 +561,12 @@ export class PhasedScriptAction extends BaseScriptAction<IPhasedCommandConfig> i
           if (!buildCacheConfiguration?.buildCacheEnabled) {
             throw new Error('You must have build cache enabled to use this option.');
           }
+
           const { BuildPlanPlugin } = await import('../../logic/operations/BuildPlanPlugin');
           new BuildPlanPlugin(terminal).apply(this.hooks);
         }
 
-        const { configuration: experiments } = this.rushConfiguration.experimentsConfiguration;
-        if (this.rushConfiguration?.isPnpm && experiments?.usePnpmSyncForInjectedDependencies) {
+        if (isPnpm && usePnpmSyncForInjectedDependencies) {
           const { PnpmSyncCopyOperationPlugin } = await import(
             '../../logic/operations/PnpmSyncCopyOperationPlugin'
           );

libraries/rush-lib/src/logic/buildCache/FileSystemBuildCacheProvider.ts

@@ -1,8 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved. Licensed under the MIT license.
 // See LICENSE in the project root for license information.
 
-import * as path from 'node:path';
-
 import { FileSystem } from '@rushstack/node-core-library';
 import type { ITerminal } from '@rushstack/terminal';
 
@@ -32,19 +30,17 @@ const DEFAULT_BUILD_CACHE_FOLDER_NAME: string = 'build-cache';
  * @beta
  */
 export class FileSystemBuildCacheProvider {
-  private readonly _cacheFolderPath: string;
-
-  public constructor(options: IFileSystemBuildCacheProviderOptions) {
-    this._cacheFolderPath =
-      options.rushUserConfiguration.buildCacheFolder ||
-      path.join(options.rushConfiguration.commonTempFolder, DEFAULT_BUILD_CACHE_FOLDER_NAME);
-  }
-
   /**
    * Returns the absolute disk path for the specified cache id.
    */
-  public getCacheEntryPath(cacheId: string): string {
-    return path.join(this._cacheFolderPath, cacheId);
+  public readonly getCacheEntryPath: (cacheId: string) => string;
+
+  public constructor(options: IFileSystemBuildCacheProviderOptions) {
+    const {
+      rushConfiguration: { commonTempFolder },
+      rushUserConfiguration: { buildCacheFolder = `${commonTempFolder}/${DEFAULT_BUILD_CACHE_FOLDER_NAME}` }
+    } = options;
+    this.getCacheEntryPath = (cacheId: string) => `${buildCacheFolder}/${cacheId}`;
   }
 
   /**
@@ -55,7 +51,8 @@ export class FileSystemBuildCacheProvider {
     cacheId: string
   ): Promise<string | undefined> {
     const cacheEntryFilePath: string = this.getCacheEntryPath(cacheId);
-    if (await FileSystem.existsAsync(cacheEntryFilePath)) {
+    const cacheEntryExists: boolean = await FileSystem.existsAsync(cacheEntryFilePath);
+    if (cacheEntryExists) {
       return cacheEntryFilePath;
     } else {
       return undefined;

libraries/rush-lib/src/logic/buildCache/ICloudBuildCacheProvider.ts

@@ -11,6 +11,35 @@ export interface ICloudBuildCacheProvider {
 
   tryGetCacheEntryBufferByIdAsync(terminal: ITerminal, cacheId: string): Promise<Buffer | undefined>;
   trySetCacheEntryBufferAsync(terminal: ITerminal, cacheId: string, entryBuffer: Buffer): Promise<boolean>;
+
+  /**
+   * If implemented, the build cache will prefer to use this method over
+   * {@link ICloudBuildCacheProvider.tryGetCacheEntryBufferByIdAsync} to avoid loading the entire
+   * cache entry into memory, if possible. The implementation should download the cache entry and write it
+   * to the specified local file path.
+   *
+   * @returns `true` if the cache entry was found and written to the file, `false` if it was
+   * not found. Throws on errors.
+   */
+  tryDownloadCacheEntryToFileAsync?(
+    terminal: ITerminal,
+    cacheId: string,
+    localFilePath: string
+  ): Promise<boolean>;
+  /**
+   * If implemented, the build cache will prefer to use this method over
+   * {@link ICloudBuildCacheProvider.trySetCacheEntryBufferAsync} to avoid loading the entire
+   * cache entry into memory, if possible. The implementation should read the cache entry from
+   * the specified local file path and upload it.
+   *
+   * @returns `true` if the cache entry was written to the cache, otherwise `false`.
+   */
+  tryUploadCacheEntryFromFileAsync?(
+    terminal: ITerminal,
+    cacheId: string,
+    localFilePath: string
+  ): Promise<boolean>;
+
   updateCachedCredentialAsync(terminal: ITerminal, credential: string): Promise<void>;
   updateCachedCredentialInteractiveAsync(terminal: ITerminal): Promise<void>;
   deleteCachedCredentialsAsync(terminal: ITerminal): Promise<void>;