[rush] feat(cobuilds): allow orchestration without using the build cache (#4871)

* feat(cobuilds): allow orchestration without using the build cache

* update comment

* rename allowCobuildOrchestration to allowCobuildWithoutCache

* move to experiments.json

* fix lint errors

* Update build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/e/config/rush-project.json

* adding to rush-init and adjusting description

---------

Co-authored-by: Aramis Sennyey <[email protected]>

Author: aramissennyeydd

build-tests/rush-redis-cobuild-plugin-integration-test/README.md

@@ -150,3 +150,24 @@ rm -rf common/temp/build-cache
 5. Open command palette (Ctrl+Shift+P or Command+Shift+P) and select `Tasks: Run Task` and select `cobuild`.
 
 Expected behavior: Cobuild feature is enabled, cobuild related logs out in both terminals. These two cobuild commands fail because of the failing build of project "A". And, one of them restored the failing build cache created by the other one.
+
+#### Case 5: Sharded cobuilds
+
+Enable the `allowCobuildWithoutCache` experiment in `experiments.json`.
+
+Navigate to the sandbox for sharded cobuilds,
+```sh
+cd sandbox/sharded-repo
+```
+
+Next, start up your Redis instance,
+```sh
+docker compose down && docker compose up -d
+```
+
+Then, open 2 terminals and run this in each (changing the RUSH_COBUILD_RUNNER_ID across the 2 terminals),
+```sh
+rm -rf common/temp/build-cache && RUSH_COBUILD_CONTEXT_ID=foo REDIS_PASS=redis123 RUSH_COBUILD_RUNNER_ID=runner1 node ../../lib/runRush.js cobuild -p 10 --timeline
+```
+
+If all goes well, you should see a bunch of operation with `- shard xx/yy`. Operations `h (build)` and `e (build)` are both sharded heavily and should be cobuild compatible. To validate changes you're making, ensure that the timeline view for all of the shards of those 2 operations are cobuilt across both terminals. If they're not, something is wrong with your update.

build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/sharded-repo/common/config/rush/experiments.json

@@ -3,7 +3,7 @@
  * Rush features.  More documentation is available on the Rush website: https://rushjs.io
  */
 {
-  "$schema": "https://developer.microsoft.com/json-schemas/rush/v5/experiments.schema.json",
+  "$schema": "../../../../../../../libraries/rush-lib/src/schemas/experiments.schema.json",
 
   /**
    * By default, 'rush install' passes --no-prefer-frozen-lockfile to 'pnpm install'.
@@ -40,7 +40,7 @@
    * If true, the phased commands feature is enabled. To use this feature, create a "phased" command
    * in common/config/rush/command-line.json.
    */
-  "phasedCommands": true
+  "phasedCommands": true,
 
   /**
    * If true, perform a clean install after when running `rush install` or `rush update` if the
@@ -52,4 +52,6 @@
    * If true, print the outputs of shell commands defined in event hooks to the console.
    */
   // "printEventHooksOutputToConsole": true
+
+  "allowCobuildWithoutCache": true
 }

build-tests/rush-redis-cobuild-plugin-integration-test/sandbox/sharded-repo/projects/e/config/rush-project.json

@@ -1,18 +1,19 @@
 {
+  "$schema": "../../../../../../../libraries/rush-lib/src/schemas/rush-project.schema.json",
   "operationSettings": [
     {
       "operationName": "_phase:build",
       "outputFolderNames": ["dist"],
+      "allowCobuildOrchestration": true,
+      "disableBuildCacheForOperation": true,
       "sharding": {
-        "count": 75,
-        "shardOperationSettings": {
-          "weight": 10
-        }
+        "count": 75
       }
     },
     {
       "operationName": "_phase:build:shard",
-      "weight": 1
+      "weight": 10,
+      "allowCobuildOrchestration": true
     }
   ]
 }

common/changes/@microsoft/rush/sennyeya-allow-cobuild-orchestration_2024-08-08-23-11.json

@@ -0,0 +1,10 @@
+{
+  "changes": [
+    {
+      "packageName": "@microsoft/rush",
+      "comment": "Adds a new experiment 'allowCobuildWithoutCache' for cobuilds to allow uncacheable operations to benefit from cobuild orchestration without using the build cache.",
+      "type": "none"
+    }
+  ],
+  "packageName": "@microsoft/rush"
+}

common/reviews/api/rush-lib.api.md

@@ -105,6 +105,7 @@ export class CobuildConfiguration {
     readonly cobuildFeatureEnabled: boolean;
     readonly cobuildLeafProjectLogOnlyAllowed: boolean;
     readonly cobuildRunnerId: string;
+    readonly cobuildWithoutCacheAllowed: boolean;
     // (undocumented)
     createLockProviderAsync(terminal: ITerminal): Promise<void>;
     // (undocumented)
@@ -469,6 +470,7 @@ export interface IExecutionResult {
 
 // @beta
 export interface IExperimentsJson {
+    allowCobuildWithoutCache?: boolean;
     buildCacheWithAllowWarningsInSuccessfulBuild?: boolean;
     buildSkipWithAllowWarningsInSuccessfulBuild?: boolean;
     cleanInstallAfterNpmrcChanges?: boolean;
@@ -625,6 +627,7 @@ export interface IOperationRunnerContext {
 
 // @alpha (undocumented)
 export interface IOperationSettings {
+    allowCobuildWithoutCache?: boolean;
     dependsOnAdditionalFiles?: string[];
     dependsOnEnvVars?: string[];
     disableBuildCacheForOperation?: boolean;

libraries/rush-lib/assets/rush-init/common/config/rush/experiments.json

@@ -96,5 +96,11 @@
    * This ensures that important notices will be seen by anyone doing active development, since people often
    * ignore normal discussion group messages or don't know to subscribe.
    */
-   /*[LINE "HYPOTHETICAL"]*/ "rushAlerts": true
+   /*[LINE "HYPOTHETICAL"]*/ "rushAlerts": true,
+
+
+  /**
+   * When using cobuilds, this experiment allows uncacheable operations to benefit from cobuild orchestration without using the build cache.
+   */
+   /*[LINE "HYPOTHETICAL"]*/ "allowCobuildWithoutCache": true
 }

libraries/rush-lib/src/api/CobuildConfiguration.ts

@@ -69,18 +69,26 @@ export class CobuildConfiguration {
    */
   public readonly cobuildLeafProjectLogOnlyAllowed: boolean;
 
+  /**
+   * If true, operations can opt into leveraging cobuilds without restoring from the build cache.
+   *  Operations will need to us the allowCobuildWithoutCache flag to opt into this behavior per phase.
+   */
+  public readonly cobuildWithoutCacheAllowed: boolean;
+
   private _cobuildLockProvider: ICobuildLockProvider | undefined;
   private readonly _cobuildLockProviderFactory: CobuildLockProviderFactory;
   private readonly _cobuildJson: ICobuildJson;
 
   private constructor(options: ICobuildConfigurationOptions) {
-    const { cobuildJson, cobuildLockProviderFactory } = options;
+    const { cobuildJson, cobuildLockProviderFactory, rushConfiguration } = options;
 
     this.cobuildContextId = EnvironmentConfiguration.cobuildContextId;
     this.cobuildFeatureEnabled = this.cobuildContextId ? cobuildJson.cobuildFeatureEnabled : false;
     this.cobuildRunnerId = EnvironmentConfiguration.cobuildRunnerId || uuidv4();
     this.cobuildLeafProjectLogOnlyAllowed =
       EnvironmentConfiguration.cobuildLeafProjectLogOnlyAllowed ?? false;
+    this.cobuildWithoutCacheAllowed =
+      rushConfiguration.experimentsConfiguration.configuration.allowCobuildWithoutCache ?? false;
 
     this._cobuildLockProviderFactory = cobuildLockProviderFactory;
     this._cobuildJson = cobuildJson;

libraries/rush-lib/src/api/ExperimentsConfiguration.ts

@@ -106,6 +106,13 @@ export interface IExperimentsJson {
    * ignore normal discussion group messages or don't know to subscribe.
    */
   rushAlerts?: boolean;
+
+  /**
+   * Allow cobuilds without using the build cache to store previous execution info. When setting up
+   *  distributed builds, Rush will allow uncacheable projects to still leverage the cobuild feature.
+   * This is useful when you want to speed up operations that can't (or shouldn't) be cached.
+   */
+  allowCobuildWithoutCache?: boolean;
 }
 
 const _EXPERIMENTS_JSON_SCHEMA: JsonSchema = JsonSchema.fromLoadedObject(schemaJson);

libraries/rush-lib/src/api/RushProjectConfiguration.ts

@@ -134,6 +134,11 @@ export interface IOperationSettings {
    *  determined by the -p flag.
    */
   weight?: number;
+
+  /**
+   * If true, this operation can use cobuilds for orchestration without restoring build cache entries.
+   */
+  allowCobuildWithoutCache?: boolean;
 }
 
 interface IOldRushProjectJson {

libraries/rush-lib/src/logic/operations/CacheableOperationPlugin.ts

@@ -99,6 +99,25 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
           ? new DisjointSet()
           : undefined;
 
+        for (const operation of recordByOperation.keys()) {
+          if (
+            operation.settings?.allowCobuildWithoutCache &&
+            !cobuildConfiguration?.cobuildWithoutCacheAllowed
+          ) {
+            throw new Error(
+              `Operation ${operation.name} is not allowed to run without the cobuild orchestration experiment enabled. You must enable the "allowCobuildWithoutCache" experiment in experiments.json.`
+            );
+          }
+          if (
+            operation.settings?.allowCobuildWithoutCache &&
+            !operation.settings.disableBuildCacheForOperation
+          ) {
+            throw new Error(
+              `Operation ${operation.name} must have disableBuildCacheForOperation set to true when using the cobuild orchestration experiment. This is to prevent implicit cache dependencies for this operation.`
+            );
+          }
+        }
+
         await Async.forEachAsync(
           recordByOperation.keys(),
           async (operation: Operation) => {
@@ -258,7 +277,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
             phase,
             configHash,
             terminal: buildCacheTerminal,
-            operationMetadataManager
+            operationMetadataManager,
+            operation: record.operation
           });
 
           // Try to acquire the cobuild lock
@@ -359,15 +379,17 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
             if (cobuildCompletedState) {
               const { status, cacheId } = cobuildCompletedState;
 
+              if (record.operation.settings?.allowCobuildWithoutCache) {
+                // This should only be enabled if the experiment for cobuild orchestration is enabled.
+                return status;
+              }
+
               const restoreFromCacheSuccess: boolean = await restoreCacheAsync(
                 cobuildLock.projectBuildCache,
                 cacheId
               );
 
               if (restoreFromCacheSuccess) {
-                if (cobuildCompletedState) {
-                  return cobuildCompletedState.status;
-                }
                 return status;
               }
             } else if (!buildCacheContext.isCacheReadAttempted && buildCacheContext.isCacheReadAllowed) {
@@ -569,7 +591,8 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
     phase,
     configHash,
     terminal,
-    operationMetadataManager
+    operationMetadataManager,
+    operation
   }: {
     buildCacheContext: IOperationBuildCacheContext;
     buildCacheConfiguration: BuildCacheConfiguration | undefined;
@@ -578,10 +601,11 @@ export class CacheableOperationPlugin implements IPhasedCommandPlugin {
     configHash: string;
     terminal: ITerminal;
     operationMetadataManager: OperationMetadataManager | undefined;
+    operation: Operation;
   }): Promise<ProjectBuildCache | undefined> {
     if (!buildCacheContext.projectBuildCache) {
       const { cacheDisabledReason } = buildCacheContext;
-      if (cacheDisabledReason) {
+      if (cacheDisabledReason && !operation.settings?.allowCobuildWithoutCache) {
         terminal.writeVerboseLine(cacheDisabledReason);
         return;
       }
@@ -863,7 +887,7 @@ export function clusterOperations(
   for (const [operation, { cacheDisabledReason }] of operationBuildCacheMap) {
     const { associatedProject: project, associatedPhase: phase } = operation;
     if (project && phase) {
-      if (cacheDisabledReason) {
+      if (cacheDisabledReason && !operation.settings?.allowCobuildWithoutCache) {
         /**
          * Group the project build cache disabled with its consumers. This won't affect too much in
          * a monorepo with high build cache coverage.