Add RedisSearch BM25 workspace query

Author: rowantrollope

AGENTS.md

@@ -235,3 +235,24 @@ The most important implementation seams are:
 - Control-plane import/checkpoint JSON fields typed as `map[string][]byte`
   expect base64-encoded string values. Small synthetic manifests can avoid that
   by using `ManifestEntry.inline`, which is already base64 text.
+- `afs status` can show running mounts from multiple product modes/databases.
+  When a command scopes to the current config, explain when a status-visible
+  mount belongs to another config instead of returning a bare "does not exist".
+- Top-level filesystem shortcuts like `afs grep` and `afs query` must route
+  through the `afs fs` command path. Do not send them to older local-only
+  helpers, or Cloud/Self-managed users will see local Redis errors instead of
+  control-plane behavior.
+- Top-level filesystem shortcut help must say shortcuts use the "default"
+  workspace and that explicit targeting requires `afs fs <workspace> <command>`.
+- Sync mount cold-start in Cloud-managed mode must hydrate from the workspace
+  session's storage key/head checkpoint. Do not require direct Redis workspace
+  metadata lookup by display name; cloud session Redis may expose the live root
+  while metadata lives behind the control plane.
+- Preserve the two-command search UX: `grep` is exact text evidence and `query`
+  is the powerful ranked retrieval command. `query` defaults to hybrid + rerank,
+  supports `lex:`, `vec:`, `hyde:`, and `intent:` documents, and uses
+  `--keyword` / `--semantic` for narrower modes. Do not reintroduce public
+  `search` or `vsearch` commands unless the product direction changes again.
+- Workspace file/query CLI calls use resolved workspace routes under
+  `/v1/workspaces/<id>/...`; when adding a scoped database route, add the
+  matching resolved route and a regression test for workspace IDs.

README.md

@@ -83,6 +83,53 @@ The important pattern is:
 
 So AFS is not trying to beat your local SSD, and it is not trying to out-POSIX mature shared filesystems. It is trying to give you a remote, checkpointable, forkable workspace with enough performance that normal tools still feel normal.
 
+## RedisSearch BM25 Query
+
+AFS workspaces are not just mounted folders. They are now queryable knowledge
+surfaces backed by RedisSearch BM25.
+
+When you mount a workspace, text files flow into the live Redis-backed
+workspace. AFS builds a RedisSearch chunk index over Markdown, JSON, JSONL,
+source files, config files, notes, logs, and other text content. Then
+`afs query` ranks the most relevant chunks with BM25:
+
+```bash
+afs query "how do checkpoints work?"
+afs query "redis connection refused during setup"
+afs query "where is auth token configuration handled?"
+afs query "ralph loops"
+```
+
+Example output:
+
+```text
+#1 /docs/checkpoints.md:12-20  score 0.42
+  Checkpoints are explicit snapshots of workspace state. File edits update the
+  live workspace immediately, and checkpoint_create records a recoverable point.
+
+#2 /README.md:148-156  score 0.31
+  grep is for exact text evidence. query is for ranked conceptual search...
+```
+
+This is useful for agents because they often know what they are looking for
+conceptually, but not the exact phrase or filename. `grep` is still the right
+tool for exact evidence. `query` is the right tool when an agent needs ranked
+context from docs, history, config, and source files.
+
+The path is:
+
+```text
+mounted folder -> AFS live workspace -> RedisSearch chunk index -> BM25 ranked results
+```
+
+If RedisSearch is unavailable or the projection is temporarily stale, AFS falls
+back to direct keyword ranking over the workspace content. Use `--explain` to
+see which backend answered a query:
+
+```bash
+afs query --explain --json "ralph loops"
+```
+
 ## Requirements
 
 AFS requires a Redis instance you provide.
@@ -142,6 +189,19 @@ If you want to save a known-good point:
 ./afs cp create my-repo before-refactor
 ```
 
+If you want to search workspace contents:
+
+```bash
+./afs fs my-repo grep "DirtyHint"
+./afs fs my-repo query "how do checkpoints work?"
+./afs fs my-repo query --keyword "checkpoint savepoint"
+```
+
+`grep` is for exact text evidence. `query` is for ranked conceptual search and
+falls back to keyword-ranked results when embeddings are disabled or
+unavailable. Keyword ranking uses RedisSearch BM25 query chunks when available,
+then falls back to direct content ranking.
+
 If you want commands with an optional workspace argument to use `my-repo` by
 default:
 
@@ -224,8 +284,9 @@ client on demand. A minimal config looks like:
 }
 ```
 
-The MCP surface is workspace-oriented: list/create/fork workspaces,
-read and edit files, grep a workspace, and create or restore checkpoints.
+The MCP surface is workspace-oriented: list/create/fork workspaces, read and
+edit files, grep exact text with `file_grep`, run ranked conceptual search
+with `file_query`, and create or restore checkpoints.
 File-edit tools update the live workspace state and leave the workspace dirty
 until `checkpoint_create` is called explicitly.
 

cmd/afs/afs_commands.go

@@ -1111,7 +1111,8 @@ func promptWorkspaceSelectionFromSummariesWithReader(workspaces []workspaceSumma
 	fmt.Println()
 	fmt.Println("Select workspace")
 	fmt.Println()
-	printPlainTable([]string{"#", "Workspace", "Workspace ID", "Database", "Updated", "Mounted"}, checkpointWorkspacePromptRows(workspaces, workspaceListMounts(workspaces)))
+	headers := []string{"#", "Workspace", "Workspace ID", "Database", "Updated", "Mounted"}
+	printPlainTable(headers, checkpointWorkspacePromptRows(workspaces, workspaceListMounts(workspaces)))
 	fmt.Println()
 	fmt.Print("Workspace: ")
 

cmd/afs/afs_commands_test.go

@@ -12,15 +12,23 @@ import (
 
 	"github.com/alicebob/miniredis/v2"
 	"github.com/redis/agent-filesystem/internal/controlplane"
+	"github.com/redis/agent-filesystem/internal/mcptools"
 	mountclient "github.com/redis/agent-filesystem/mount/client"
 	"github.com/redis/go-redis/v9"
 )
 
 type stubAFSControlPlane struct {
-	workspaces controlplane.WorkspaceListResponse
+	workspaces         controlplane.WorkspaceListResponse
+	workspacesErr      error
+	workspaceConfig    controlplane.WorkspaceConfig
+	workspaceConfigOK  bool
+	workspaceConfigErr error
 }
 
 func (s stubAFSControlPlane) ListWorkspaceSummaries(context.Context) (controlplane.WorkspaceListResponse, error) {
+	if s.workspacesErr != nil {
+		return controlplane.WorkspaceListResponse{}, s.workspacesErr
+	}
 	return s.workspaces, nil
 }
 
@@ -29,6 +37,12 @@ func (s stubAFSControlPlane) GetWorkspace(context.Context, string) (controlplane
 }
 
 func (s stubAFSControlPlane) GetWorkspaceConfig(context.Context, string) (controlplane.WorkspaceConfig, error) {
+	if s.workspaceConfigErr != nil {
+		return controlplane.WorkspaceConfig{}, s.workspaceConfigErr
+	}
+	if s.workspaceConfigOK {
+		return s.workspaceConfig, nil
+	}
 	return controlplane.WorkspaceConfig{}, fmt.Errorf("unexpected GetWorkspaceConfig call")
 }
 
@@ -112,6 +126,18 @@ func (s stubAFSControlPlane) GetFileContent(context.Context, string, string, str
 	return controlplane.FileContentResponse{}, fmt.Errorf("unexpected GetFileContent call")
 }
 
+func (s stubAFSControlPlane) QueryWorkspace(context.Context, string, mcptools.FileQueryRequest) (mcptools.FileQueryResponse, error) {
+	return mcptools.FileQueryResponse{}, fmt.Errorf("unexpected QueryWorkspace call")
+}
+
+func (s stubAFSControlPlane) QueryIndexStatus(context.Context, string, controlplane.WorkspaceQueryIndexStatusRequest) (controlplane.WorkspaceQueryIndexStatus, error) {
+	return controlplane.WorkspaceQueryIndexStatus{}, fmt.Errorf("unexpected QueryIndexStatus call")
+}
+
+func (s stubAFSControlPlane) RebuildQueryIndex(context.Context, string, controlplane.WorkspaceQueryIndexRebuildRequest) (controlplane.WorkspaceQueryIndexRebuildResponse, error) {
+	return controlplane.WorkspaceQueryIndexRebuildResponse{}, fmt.Errorf("unexpected RebuildQueryIndex call")
+}
+
 func (s stubAFSControlPlane) DiffWorkspace(context.Context, string, string, string) (controlplane.WorkspaceDiffResponse, error) {
 	return controlplane.WorkspaceDiffResponse{}, fmt.Errorf("unexpected DiffWorkspace call")
 }
@@ -495,6 +521,90 @@ func TestResolveWorkspaceSelectionPrefersCWDMountedWorkspace(t *testing.T) {
 	}
 }
 
+func TestResolveWorkspaceSetDefaultSelectionUsesCurrentConfigMountWhenCatalogMissesWorkspace(t *testing.T) {
+	t.Helper()
+
+	homeDir := withTempHome(t)
+	cfg := defaultConfig()
+	cfg.ProductMode = productModeCloud
+	cfg.URL = "https://afs.cloud"
+	cfg.DatabaseID = "afs-cloud"
+
+	if err := saveMountRegistry(mountRegistry{Mounts: []mountRecord{{
+		ID:                   "mnt_first",
+		Workspace:            "first-workspace",
+		WorkspaceID:          "ws_first",
+		LocalPath:            filepath.Join(homeDir, "first-workspace"),
+		ProductMode:          productModeCloud,
+		ControlPlaneURL:      cfg.URL,
+		ControlPlaneDatabase: cfg.DatabaseID,
+		PID:                  os.Getpid(),
+		Mode:                 modeSync,
+		StartedAt:            time.Now().UTC(),
+	}}}); err != nil {
+		t.Fatalf("saveMountRegistry() returned error: %v", err)
+	}
+
+	selection, err := resolveWorkspaceSetDefaultSelection(context.Background(), cfg, stubAFSControlPlane{
+		workspaces: controlplane.WorkspaceListResponse{
+			Items: []controlplane.WorkspaceSummary{
+				{ID: "ws_new", Name: "new", DatabaseID: cfg.DatabaseID},
+			},
+		},
+	}, "first-workspace")
+	if err != nil {
+		t.Fatalf("resolveWorkspaceSetDefaultSelection() returned error: %v", err)
+	}
+	if selection.ID != "ws_first" || selection.Name != "first-workspace" || selection.Source != workspaceSelectionExplicit {
+		t.Fatalf("selection = %+v, want mounted first-workspace", selection)
+	}
+}
+
+func TestResolveWorkspaceSetDefaultSelectionExplainsMountedWorkspaceFromOtherConfig(t *testing.T) {
+	t.Helper()
+
+	homeDir := withTempHome(t)
+	cfg := defaultConfig()
+	cfg.ProductMode = productModeCloud
+	cfg.URL = "https://afs.cloud"
+	cfg.DatabaseID = "afs-cloud"
+
+	if err := saveMountRegistry(mountRegistry{Mounts: []mountRecord{{
+		ID:                   "mnt_first",
+		Workspace:            "first-workspace",
+		WorkspaceID:          "ws_first",
+		LocalPath:            filepath.Join(homeDir, "first-workspace"),
+		ProductMode:          productModeSelfHosted,
+		ControlPlaneURL:      "http://127.0.0.1:8091",
+		ControlPlaneDatabase: "localhost-6379",
+		PID:                  os.Getpid(),
+		Mode:                 modeSync,
+		StartedAt:            time.Now().UTC(),
+	}}}); err != nil {
+		t.Fatalf("saveMountRegistry() returned error: %v", err)
+	}
+
+	_, err := resolveWorkspaceSetDefaultSelection(context.Background(), cfg, stubAFSControlPlane{
+		workspaces: controlplane.WorkspaceListResponse{
+			Items: []controlplane.WorkspaceSummary{
+				{ID: "ws_new", Name: "new", DatabaseID: cfg.DatabaseID},
+			},
+		},
+	}, "first-workspace")
+	if err == nil {
+		t.Fatal("resolveWorkspaceSetDefaultSelection() returned nil error, want config mismatch")
+	}
+	for _, want := range []string{
+		`workspace "first-workspace" is mounted from Self-managed http://127.0.0.1:8091 (localhost-6379)`,
+		"current config uses Cloud-managed https://afs.cloud (afs-cloud)",
+		"status --verbose",
+	} {
+		if !strings.Contains(err.Error(), want) {
+			t.Fatalf("error = %q, want %q", err, want)
+		}
+	}
+}
+
 func TestResolveWorkspaceSelectionUsesOnlyMountedWorkspace(t *testing.T) {
 	t.Helper()
 

cmd/afs/afs_mcp.go

@@ -405,7 +405,6 @@ func (s *afsMCPServer) Tools(_ context.Context) []mcpproto.Tool {
 					"path":      map[string]string{"type": "string", "description": "Absolute workspace directory path to search within, for example / or /src", "default": "/"},
 					"pattern":   map[string]string{"type": "string", "description": "Basename glob pattern, for example *.go or [Mm]akefile"},
 					"kind":      map[string]string{"type": "string", "description": "Optional kind filter: file, dir, symlink, or any", "default": "file"},
-					"limit":     map[string]string{"type": "integer", "description": "Maximum number of results to return", "default": "100"},
 				},
 				"required": []string{"pattern"},
 			},
@@ -536,6 +535,39 @@ func (s *afsMCPServer) Tools(_ context.Context) []mcpproto.Tool {
 				"required": []string{"pattern"},
 			},
 		},
+		{
+			Name:        "file_query",
+			Description: "Rank workspace files for a concept or natural-language question. Use this when exact text is unknown or when QMD-style lex/vec/hyde clauses are useful. Use file_grep instead for deterministic exact line matches.",
+			InputSchema: map[string]any{
+				"type": "object",
+				"properties": map[string]any{
+					"workspace": map[string]string{"type": "string", "description": "Workspace name"},
+					"path":      map[string]string{"type": "string", "description": "Absolute workspace path to search within, for example / or /src", "default": "/"},
+					"query":     map[string]string{"type": "string", "description": "Plain query text or a QMD-style typed query document using lex:, vec:, hyde:, and intent:"},
+					"mode":      map[string]string{"type": "string", "description": "query (default), keyword, or semantic", "default": "query"},
+					"searches": map[string]any{
+						"type":        "array",
+						"description": "Optional typed searches for mode=query",
+						"items": map[string]any{
+							"type": "object",
+							"properties": map[string]any{
+								"type":  map[string]string{"type": "string", "description": "lex, vec, or hyde"},
+								"query": map[string]string{"type": "string", "description": "Query text for this clause"},
+							},
+							"required": []string{"type", "query"},
+						},
+					},
+					"intent":          map[string]string{"type": "string", "description": "Extra retrieval intent"},
+					"limit":           map[string]string{"type": "integer", "description": "Maximum results", "default": "10"},
+					"all":             map[string]string{"type": "boolean", "description": "Return all results"},
+					"min_score":       map[string]string{"type": "number", "description": "Minimum score"},
+					"candidate_limit": map[string]string{"type": "integer", "description": "Candidate result limit"},
+					"rerank":          map[string]string{"type": "string", "description": "auto or none", "default": "auto"},
+					"explain":         map[string]string{"type": "boolean", "description": "Include retrieval explanation"},
+					"chunk_strategy":  map[string]string{"type": "string", "description": "auto or regex"},
+				},
+			},
+		},
 	}
 	filtered := make([]mcpproto.Tool, 0, len(tools))
 	for _, tool := range tools {
@@ -612,6 +644,8 @@ func (s *afsMCPServer) CallTool(ctx context.Context, name string, args map[strin
 		value, err = s.toolFilePatch(ctx, args)
 	case "file_grep":
 		value, err = s.toolFileGrep(ctx, args)
+	case "file_query":
+		value, err = s.toolFileQuery(ctx, args)
 	default:
 		err = fmt.Errorf("unknown tool %q", name)
 	}
@@ -1221,14 +1255,6 @@ func (s *afsMCPServer) toolFileGlob(ctx context.Context, args map[string]any) (a
 	default:
 		return nil, fmt.Errorf("argument %q must be one of file, dir, symlink, or any", "kind")
 	}
-	limit, err := mcpInt(args, "limit", 100)
-	if err != nil {
-		return nil, err
-	}
-	if limit <= 0 {
-		limit = 100
-	}
-
 	fsKey, _, _, err := s.store.ensureWorkspaceRoot(ctx, workspace)
 	if err != nil {
 		return nil, err
@@ -1252,11 +1278,6 @@ func (s *afsMCPServer) toolFileGlob(ctx context.Context, args map[string]any) (a
 	if err != nil {
 		return nil, err
 	}
-	truncated := false
-	if len(matches) > limit {
-		truncated = true
-		matches = matches[:limit]
-	}
 	items := make([]mcpFileListItem, 0, len(matches))
 	for _, matchPath := range matches {
 		item, err := workspaceFileListItem(ctx, fsClient, matchPath)
@@ -1282,7 +1303,6 @@ func (s *afsMCPServer) toolFileGlob(ctx context.Context, args map[string]any) (a
 		"pattern":   pattern,
 		"kind":      ternaryString(kind == "", "any", kind),
 		"count":     len(items),
-		"truncated": truncated,
 		"items":     items,
 	}, nil
 }
@@ -1695,6 +1715,21 @@ func (s *afsMCPServer) toolFileGrep(ctx context.Context, args map[string]any) (a
 	return result, nil
 }
 
+func (s *afsMCPServer) toolFileQuery(ctx context.Context, args map[string]any) (any, error) {
+	workspace, err := s.resolveWorkspaceArg(ctx, args, "workspace")
+	if err != nil {
+		return nil, err
+	}
+	request, err := mcptools.FileQueryRequestFromArgs(args, workspace)
+	if err != nil {
+		return nil, err
+	}
+	request.Workspace = workspace
+	request.Path = normalizeAFSGrepPath(request.Path)
+
+	return s.service.QueryWorkspace(ctx, workspace, request)
+}
+
 func (s *afsMCPServer) grepLocalWorkspace(ctx context.Context, workspace, searchPath string, opts grepOptions) (any, error) {
 	workspaceMeta, err := s.store.getWorkspaceMeta(ctx, workspace)
 	if err != nil {