refactor: ♻️ Refactor CodeBaseIndexer out of core

Author: chezsmithy

core/config/yaml/loadYaml.ts

@@ -68,7 +68,7 @@ function convertYamlMcpToContinueMcp(
   return {
     transport: {
       type: "stdio",
-      command: server.command,
+      command: server.command || "", // Add default empty string to handle undefined
       args: server.args ?? [],
       env: server.env,
     } as any, // TODO: Fix the mcpServers types in config-yaml (discriminated union)
@@ -452,16 +452,28 @@ async function configYamlToContinueConfig(options: {
   // Trigger MCP server refreshes (Config is reloaded again once connected!)
   const mcpManager = MCPManagerSingleton.getInstance();
   mcpManager.setConnections(
-    (config.mcpServers ?? []).map((server) => ({
-      id: server.name,
-      name: server.name,
-      transport: {
-        type: "stdio",
-        args: [],
-        ...(server as any), // TODO: fix the types on mcpServers in config-yaml
-      },
-      timeout: server.connectionTimeout,
-    })),
+    (config.mcpServers ?? []).map((server) => {
+      // Create proper transport object based on type
+      const transport = server.type === "sse"
+        ? {
+            type: "sse" as const,
+            url: server.url || "",
+            faviconUrl: server.faviconUrl
+          }
+        : {
+            type: "stdio" as const,
+        command: server.command || "",
+            args: server.args || [],
+            env: server.env,
+            faviconUrl: server.faviconUrl
+          };
+    return {
+        id: server.name,
+        name: server.name,
+        transport,
+        timeout: server.connectionTimeout
+    };
+    }),
     false,
   );
 
@@ -507,8 +519,8 @@ export async function loadContinueConfigFromYaml(options: {
       errors: configYamlResult.errors,
       config: undefined,
       configLoadInterrupted: true,
-    };
-  }
+  };
+}
 
   const { config: continueConfig, errors: localErrors } =
     await configYamlToContinueConfig({

core/core.ts

@@ -14,7 +14,7 @@ import { getAuthUrlForTokenPage } from "./control-plane/auth/index";
 import { getControlPlaneEnv } from "./control-plane/env";
 import { DevDataSqliteDb } from "./data/devdataSqlite";
 import { DataLogger } from "./data/log";
-import { CodebaseIndexer, PauseToken } from "./indexing/CodebaseIndexer";
+import { CodebaseIndexer } from "./indexing/CodebaseIndexer";
 import DocsService from "./indexing/docs/DocsService";
 import { countTokens } from "./llm/countTokens";
 import Ollama from "./llm/llms/Ollama";
@@ -40,8 +40,7 @@ import {
   RangeInFile,
   type ContextItem,
   type ContextItemId,
-  type IDE,
-  type IndexingProgressUpdate,
+  type IDE
 } from ".";
 
 import { ConfigYaml } from "@continuedev/config-yaml";
@@ -56,7 +55,6 @@ import { MCPManagerSingleton } from "./context/mcp/MCPManagerSingleton";
 import { streamDiffLines } from "./edit/streamDiffLines";
 import { shouldIgnore } from "./indexing/shouldIgnore";
 import { walkDirCache } from "./indexing/walkDir";
-import { LLMError } from "./llm";
 import { LLMLogger } from "./llm/logger";
 import { llmStreamChat } from "./llm/streamChat";
 import type { FromCoreProtocol, ToCoreProtocol } from "./protocol";
@@ -65,18 +63,13 @@ import { StreamAbortManager } from "./util/abortManager";
 
 export class Core {
   configHandler: ConfigHandler;
-  codebaseIndexerPromise: Promise<CodebaseIndexer>;
+  codeBaseIndexer: CodebaseIndexer;
   completionProvider: CompletionProvider;
-  continueServerClientPromise: Promise<ContinueServerClient>;
-  codebaseIndexingState: IndexingProgressUpdate;
+   continueServerClientPromise: Promise<ContinueServerClient>;
   private docsService: DocsService;
   private globalContext = new GlobalContext();
   llmLogger = new LLMLogger();
 
-  private readonly indexingPauseToken = new PauseToken(
-    this.globalContext.get("indexingPaused") === true,
-  );
-
   private messageAbortControllers = new Map<string, AbortController>();
   private addMessageAbortController(id: string): AbortController {
     const controller = new AbortController();
@@ -114,12 +107,6 @@ export class Core {
     // Ensure .continue directory is created
     migrateV1DevDataFiles();
 
-    this.codebaseIndexingState = {
-      status: "loading",
-      desc: "loading",
-      progress: 0,
-    };
-
     const ideInfoPromise = messenger.request("getIdeInfo", undefined);
     const ideSettingsPromise = messenger.request("getIdeSettings", undefined);
     const sessionInfoPromise = messenger.request("getControlPlaneSessionInfo", {
@@ -144,6 +131,13 @@ export class Core {
       await this.configHandler.reloadConfig();
     };
 
+    this.codeBaseIndexer = new CodebaseIndexer(
+      this.configHandler,
+      this.ide,
+      this.messenger,
+      this.globalContext.get("indexingPaused"),
+    );
+
     this.configHandler.onConfigUpdate(async (result) => {
       const serializedResult = await this.configHandler.getSerializedConfig();
       this.messenger.send("configUpdate", {
@@ -170,38 +164,24 @@ export class Core {
     dataLogger.ideInfoPromise = ideInfoPromise;
     dataLogger.ideSettingsPromise = ideSettingsPromise;
 
-    // Codebase Indexer and ContinueServerClient depend on IdeSettings
-    let codebaseIndexerResolve: (_: any) => void | undefined;
-    this.codebaseIndexerPromise = new Promise(
-      async (resolve) => (codebaseIndexerResolve = resolve),
-    );
-
     let continueServerClientResolve: (_: any) => void | undefined;
     this.continueServerClientPromise = new Promise(
       (resolve) => (continueServerClientResolve = resolve),
     );
 
+
     void ideSettingsPromise.then((ideSettings) => {
-      const continueServerClient = new ContinueServerClient(
+            const continueServerClient = new ContinueServerClient(
         ideSettings.remoteConfigServerUrl,
         ideSettings.userToken,
       );
       continueServerClientResolve(continueServerClient);
 
-      codebaseIndexerResolve(
-        new CodebaseIndexer(
-          this.configHandler,
-          this.ide,
-          this.indexingPauseToken,
-          continueServerClient,
-        ),
-      );
-
       // Index on initialization
       void this.ide.getWorkspaceDirs().then(async (dirs) => {
         // Respect pauseCodebaseIndexOnStart user settings
         if (ideSettings.pauseCodebaseIndexOnStart) {
-          this.indexingPauseToken.paused = true;
+          this.codeBaseIndexer.paused = true;
           void this.messenger.request("indexProgress", {
             progress: 0,
             desc: "Initial Indexing Skipped",
@@ -210,7 +190,7 @@ export class Core {
           return;
         }
 
-        void this.refreshCodebaseIndex(dirs);
+        await this.codeBaseIndexer.refreshCodebaseIndex(dirs);
       });
     });
 
@@ -551,25 +531,24 @@ export class Core {
       }
       walkDirCache.invalidate();
       if (data?.shouldClearIndexes) {
-        const codebaseIndexer = await this.codebaseIndexerPromise;
-        await codebaseIndexer.clearIndexes();
+        await this.codeBaseIndexer.clearIndexes();
       }
 
       const dirs = data?.dirs ?? (await this.ide.getWorkspaceDirs());
-      await this.refreshCodebaseIndex(dirs);
+      await this.codeBaseIndexer.refreshCodebaseIndex(dirs);
     });
     on("index/setPaused", (msg) => {
       this.globalContext.update("indexingPaused", msg.data);
-      this.indexingPauseToken.paused = msg.data;
+      // Update using the new setter instead of token
+      this.codeBaseIndexer.paused = msg.data;
     });
     on("index/indexingProgressBarInitialized", async (msg) => {
       // Triggered when progress bar is initialized.
       // If a non-default state has been stored, update the indexing display to that state
-      if (this.codebaseIndexingState.status !== "loading") {
-        void this.messenger.request(
-          "indexProgress",
-          this.codebaseIndexingState,
-        );
+      const currentState = this.codeBaseIndexer.currentIndexingState;
+
+      if (currentState.status !== "loading") {
+        void this.messenger.request("indexProgress", currentState);
       }
     });
 
@@ -589,7 +568,7 @@ export class Core {
         });
         const { config } = await this.configHandler.loadConfig();
         if (config && !config.disableIndexing) {
-          await this.refreshCodebaseIndexFiles(toRefresh);
+          await this.codeBaseIndexer.refreshCodebaseIndexFiles(toRefresh);
         }
       }
     };
@@ -853,7 +832,7 @@ export class Core {
             // Reindex the file
             const ignore = await shouldIgnore(uri, this.ide);
             if (!ignore) {
-              await this.refreshCodebaseIndexFiles([uri]);
+              await this.codeBaseIndexer.refreshCodebaseIndexFiles([uri]);
             }
           }
         }
@@ -1024,100 +1003,4 @@ export class Core {
       return [];
     }
   };
-
-  private indexingCancellationController: AbortController | undefined;
-  private async sendIndexingErrorTelemetry(update: IndexingProgressUpdate) {
-    console.debug(
-      "Indexing failed with error: ",
-      update.desc,
-      update.debugInfo,
-    );
-    void Telemetry.capture(
-      "indexing_error",
-      {
-        error: update.desc,
-        stack: update.debugInfo,
-      },
-      false,
-    );
-  }
-
-  private async refreshCodebaseIndex(paths: string[]) {
-    if (this.indexingCancellationController) {
-      this.indexingCancellationController.abort();
-    }
-    this.indexingCancellationController = new AbortController();
-    try {
-      for await (const update of (
-        await this.codebaseIndexerPromise
-      ).refreshDirs(paths, this.indexingCancellationController.signal)) {
-        let updateToSend = { ...update };
-
-        void this.messenger.request("indexProgress", updateToSend);
-        this.codebaseIndexingState = updateToSend;
-
-        if (update.status === "failed") {
-          void this.sendIndexingErrorTelemetry(update);
-        }
-      }
-    } catch (e: any) {
-      console.log(`Failed refreshing codebase index directories : ${e}`);
-      this.handleIndexingError(e);
-    }
-
-    this.messenger.send("refreshSubmenuItems", {
-      providers: "dependsOnIndexing",
-    });
-    this.indexingCancellationController = undefined;
-  }
-
-  private async refreshCodebaseIndexFiles(files: string[]) {
-    // Can be cancelled by codebase index but not vice versa
-    if (
-      this.indexingCancellationController &&
-      !this.indexingCancellationController.signal.aborted
-    ) {
-      return;
-    }
-    this.indexingCancellationController = new AbortController();
-    try {
-      for await (const update of (
-        await this.codebaseIndexerPromise
-      ).refreshFiles(files)) {
-        let updateToSend = { ...update };
-
-        void this.messenger.request("indexProgress", updateToSend);
-        this.codebaseIndexingState = updateToSend;
-
-        if (update.status === "failed") {
-          void this.sendIndexingErrorTelemetry(update);
-        }
-      }
-    } catch (e: any) {
-      console.log(`Failed refreshing codebase index files : ${e}`);
-      this.handleIndexingError(e);
-    }
-
-    this.messenger.send("refreshSubmenuItems", {
-      providers: "dependsOnIndexing",
-    });
-    this.indexingCancellationController = undefined;
-  }
-
-  // private
-  handleIndexingError(e: any) {
-    if (e instanceof LLMError) {
-      // Need to report this specific error to the IDE for special handling
-      void this.messenger.request("reportError", e);
-    }
-    // broadcast indexing error
-    let updateToSend: IndexingProgressUpdate = {
-      progress: 0,
-      status: "failed",
-      desc: e.message,
-    };
-    void this.messenger.request("indexProgress", updateToSend);
-    this.codebaseIndexingState = updateToSend;
-    void this.sendIndexingErrorTelemetry(updateToSend);
-  }
 }