Add semantic embeddings query support

Author: rowantrollope

AGENTS.md

@@ -247,6 +247,9 @@ The most important implementation seams are:
   control-plane behavior.
 - Top-level filesystem shortcut help must say shortcuts use the "default"
   workspace and that explicit targeting requires `afs fs <workspace> <command>`.
+- CLI error rendering must preserve help and usage blocks literally. Do not
+  sentence-format, title-case, or append punctuation to command names, flags,
+  examples, or subcommand lists.
 - Sync mount cold-start in Cloud-managed mode must hydrate from the workspace
   session's storage key/head checkpoint. Do not require direct Redis workspace
   metadata lookup by display name; cloud session Redis may expose the live root
@@ -256,6 +259,27 @@ The most important implementation seams are:
   supports `lex:`, `vec:`, `hyde:`, and `intent:` documents, and uses
   `--keyword` / `--semantic` for narrower modes. Do not reintroduce public
   `search` or `vsearch` commands unless the product direction changes again.
+- Semantic query embeddings must use a real provider. QMD uses a local GGUF
+  embedding model with explicit query/document formatting; deterministic hash
+  embeddings are acceptable only as test doubles, never as product behavior.
+- Semantic query provider/model settings are global runtime settings, not
+  workspace config. Embeddings should be treated as on by default; if provider
+  credentials are missing, return/report unavailable status without hard-failing
+  normal query flows.
+- `AFS_EMBED_MODEL`, `AFS_EMBED_PROVIDER`, `AFS_EMBED_DIMENSIONS`, and
+  `OPENAI_API_KEY` are read by the control-plane process. CLI help and
+  troubleshooting copy must not imply that setting them only on an `afs query`
+  invocation changes an already-running control plane.
+- Semantic embedding backfill must batch provider requests. Query chunks are
+  individually capped, but a large workspace can still exceed OpenAI's total
+  tokens-per-request limit if every pending chunk is embedded at once.
+- Semantic query can take longer than normal control-plane calls on first
+  provider backfill. Keep query HTTP client timeouts separate from quick
+  metadata/status calls so first-run embedding work does not fail at 30s.
+- Semantic query must not backfill embeddings as a side effect. Imports should
+  start embedding creation in the control plane when the global provider is
+  available, and existing workspaces should use an explicit query index create
+  path for embedding backfill.
 - Workspace file/query CLI calls use resolved workspace routes under
   `/v1/workspaces/<id>/...`; when adding a scoped database route, add the
   matching resolved route and a regression test for workspace IDs.

README.md

@@ -213,9 +213,16 @@ If you want to search workspace contents:
 ```
 
 `grep` is for exact text evidence. `query` is for ranked conceptual search and
-falls back to keyword-ranked results when embeddings are disabled or
-unavailable. Keyword ranking uses RedisSearch BM25 query chunks when available,
-then falls back to direct content ranking.
+currently falls back to keyword-ranked results until hybrid vector/rerank is
+complete. Keyword ranking uses RedisSearch BM25 query chunks when available,
+then falls back to direct content ranking. Use `query --semantic` for
+vector-only retrieval. Semantic embeddings are globally enabled and use OpenAI
+when `OPENAI_API_KEY` is available in the control-plane environment. Override
+the default `openai:text-embedding-3-small` model with `AFS_EMBED_MODEL` in the
+same environment, then restart the control plane. Semantic queries read
+existing embedding indexes; imports start embedding creation in the background,
+and existing workspaces can be prepared with
+`afs fs <workspace> query index create --embeddings --wait`.
 
 If you want commands with an optional workspace argument to use `my-repo` by
 default:

cmd/afs/afs_mcp_test.go

@@ -384,6 +384,7 @@ func TestAFSMCPFileQueryRanksWorkspaceContent(t *testing.T) {
 
 func TestAFSMCPFileQueryTypedClausesAndSemanticUnavailable(t *testing.T) {
 	t.Helper()
+	t.Setenv("OPENAI_API_KEY", "")
 
 	server, closeFn := setupAFSMCPTestServer(t)
 	defer closeFn()
@@ -415,8 +416,8 @@ func TestAFSMCPFileQueryTypedClausesAndSemanticUnavailable(t *testing.T) {
 	if len(response.Results) != 1 || response.Results[0].Path != "/docs/checkpoints.md" {
 		t.Fatalf("results = %#v, want scoped docs result", response.Results)
 	}
-	if len(response.Warnings) != 1 || !strings.Contains(response.Warnings[0], "Embeddings are disabled") {
-		t.Fatalf("warnings = %#v, want embeddings disabled fallback", response.Warnings)
+	if len(response.Warnings) != 1 || !strings.Contains(response.Warnings[0], "semantic clauses were keyword-ranked") {
+		t.Fatalf("warnings = %#v, want semantic-clause fallback", response.Warnings)
 	}
 
 	semantic := server.callTool(context.Background(), "file_query", map[string]any{

cmd/afs/afs_query_commands.go

@@ -80,19 +80,8 @@ func runWorkspaceQuery(mode, workspace string, args []string) error {
 	}
 	defer remote.close()
 
-	cfg, err := workspaceQueryConfig(ctx, remote.controlPlane, remote.selection)
-	if err != nil {
-		return err
-	}
 	request := workspaceQueryRequest(remote.selection, opts)
-	if opts.mode == mcptools.FileQueryModeKeyword || opts.mode == mcptools.FileQueryModeHybrid {
-		return runWorkspaceKeywordQuery(ctx, remote, opts, request, cfg)
-	}
-	message := workspaceQueryUnavailableMessage(opts, remote.selection.Name, cfg)
-	if opts.jsonOut {
-		return encodeWorkspaceQueryUnavailable(request, message)
-	}
-	return errors.New(message)
+	return runWorkspaceQueryRequest(ctx, remote, opts, request)
 }
 
 func isWorkspaceQueryIndexInvocation(args []string) bool {
@@ -103,7 +92,7 @@ func isWorkspaceQueryIndexInvocation(args []string) bool {
 		return true
 	}
 	switch strings.TrimSpace(args[1]) {
-	case "status", "rebuild", "clean":
+	case "status", "create", "rebuild", "clean":
 		return true
 	default:
 		return false
@@ -293,49 +282,30 @@ func workspaceQueryRequest(selection workspaceSelection, opts workspaceQueryOpti
 	return request
 }
 
-func runWorkspaceKeywordQuery(ctx context.Context, remote *fsRemoteWorkspace, opts workspaceQueryOptions, request mcptools.FileQueryRequest, cfg controlplane.WorkspaceConfig) error {
+func runWorkspaceQueryRequest(ctx context.Context, remote *fsRemoteWorkspace, opts workspaceQueryOptions, request mcptools.FileQueryRequest) error {
 	response, err := remote.controlPlane.QueryWorkspace(ctx, remote.selection.ID, request)
 	if err != nil {
 		return err
 	}
-	if !opts.jsonOut && opts.mode == mcptools.FileQueryModeHybrid && !cfg.Query.Embeddings.Enabled && !workspaceQueryHasSemanticClauses(request.Searches) {
-		fmt.Fprintln(os.Stderr, "Warning: Embeddings disabled; using keyword retrieval.")
-	}
 	return writeWorkspaceQueryResponse(response, opts)
 }
 
-func workspaceQueryHasSemanticClauses(searches []mcptools.FileQuerySearch) bool {
-	for _, search := range searches {
-		switch search.Type {
-		case mcptools.FileQuerySearchVec, mcptools.FileQuerySearchHyde:
-			return true
-		}
-	}
-	return false
-}
-
-func encodeWorkspaceQueryUnavailable(request mcptools.FileQueryRequest, message string) error {
-	response := mcptools.FileQueryResponse{
-		Status:    mcptools.FileQueryStatusUnavailable,
-		Workspace: request.Workspace,
-		Path:      request.Path,
-		Query:     request.Query,
-		Searches:  request.Searches,
-		Intent:    request.Intent,
-		Results:   []mcptools.FileQueryResult{},
-		Warnings:  []string{message},
-	}
-	enc := json.NewEncoder(os.Stdout)
-	enc.SetIndent("", "  ")
-	return enc.Encode(response)
-}
-
 func writeWorkspaceQueryResponse(response mcptools.FileQueryResponse, opts workspaceQueryOptions) error {
 	if opts.jsonOut {
 		enc := json.NewEncoder(os.Stdout)
 		enc.SetIndent("", "  ")
 		return enc.Encode(response)
 	}
+	if response.Status == mcptools.FileQueryStatusUnavailable {
+		message := "query is unavailable"
+		for _, warning := range response.Warnings {
+			if strings.TrimSpace(warning) != "" {
+				message = strings.TrimSpace(warning)
+				break
+			}
+		}
+		return errors.New(message)
+	}
 	for _, warning := range response.Warnings {
 		if strings.TrimSpace(warning) != "" {
 			fmt.Fprintf(os.Stderr, "Warning: %s\n", warning)
@@ -403,13 +373,30 @@ func writeWorkspaceQueryResponse(response mcptools.FileQueryResponse, opts works
 }
 
 func writeWorkspaceQueryFiles(response mcptools.FileQueryResponse, results []mcptools.FileQueryResult) error {
-	for _, result := range results {
+	for _, result := range workspaceQueryFileResults(results) {
 		docID := workspaceQueryResultDocID(result)
-		fmt.Fprintf(os.Stdout, "#%s,%s,%s\n", docID, workspaceQueryScoreNumber(result.Score), workspaceQueryResultURI(response.Workspace, result))
+		fmt.Fprintf(os.Stdout, "#%s,%s,%s\n", docID, workspaceQueryScoreNumber(result.Score), workspaceQueryResultFileURI(response.Workspace, result))
 	}
 	return nil
 }
 
+func workspaceQueryFileResults(results []mcptools.FileQueryResult) []mcptools.FileQueryResult {
+	out := make([]mcptools.FileQueryResult, 0, len(results))
+	positions := make(map[string]int, len(results))
+	for _, result := range results {
+		pathKey := normalizeFSRemotePath(result.Path)
+		if index, ok := positions[pathKey]; ok {
+			if result.Score > out[index].Score {
+				out[index] = result
+			}
+			continue
+		}
+		positions[pathKey] = len(out)
+		out = append(out, result)
+	}
+	return out
+}
+
 func writeWorkspaceQueryCSV(response mcptools.FileQueryResponse, results []mcptools.FileQueryResult, opts workspaceQueryOptions) error {
 	writer := csv.NewWriter(os.Stdout)
 	if err := writer.Write([]string{"docid", "score", "file", "title", "context", "line", "snippet"}); err != nil {
@@ -611,12 +598,7 @@ func workspaceQueryResultSource(result mcptools.FileQueryResult) string {
 }
 
 func workspaceQueryResultURI(workspace string, result mcptools.FileQueryResult) string {
-	workspace = strings.Trim(strings.TrimSpace(workspace), "/")
-	if workspace == "" {
-		workspace = "workspace"
-	}
-	remotePath := normalizeFSRemotePath(result.Path)
-	uri := "afs://" + workspace + remotePath
+	uri := workspaceQueryResultFileURI(workspace, result)
 	if result.StartLine > 0 {
 		if result.EndLine > result.StartLine {
 			uri += fmt.Sprintf(":%d-%d", result.StartLine, result.EndLine)
@@ -627,6 +609,16 @@ func workspaceQueryResultURI(workspace string, result mcptools.FileQueryResult)
 	return uri
 }
 
+func workspaceQueryResultFileURI(workspace string, result mcptools.FileQueryResult) string {
+	workspace = strings.Trim(strings.TrimSpace(workspace), "/")
+	if workspace == "" {
+		workspace = "workspace"
+	}
+	remotePath := normalizeFSRemotePath(result.Path)
+	uri := "afs://" + workspace + remotePath
+	return uri
+}
+
 func workspaceQueryResultDocID(result mcptools.FileQueryResult) string {
 	if id := strings.TrimSpace(result.ChunkID); id != "" {
 		return shortHash(id)
@@ -726,23 +718,6 @@ func workspaceQueryMetadataInt(metadata map[string]any, key string) (int, bool)
 	}
 }
 
-func workspaceQueryUnavailableMessage(opts workspaceQueryOptions, workspace string, cfg controlplane.WorkspaceConfig) string {
-	switch opts.mode {
-	case mcptools.FileQueryModeKeyword:
-		return fmt.Sprintf("keyword query is not ready yet for workspace %q\nIt will use BM25 ranking through RedisSearch. Until then, use '%s grep <pattern>' for exact line matches.", workspace, filepath.Base(os.Args[0]))
-	case mcptools.FileQueryModeSemantic:
-		if !cfg.Query.Embeddings.Enabled {
-			return fmt.Sprintf("semantic query is disabled for workspace %q\nEnable it with: %s ws config %s set query.embeddings.enabled true", workspace, filepath.Base(os.Args[0]), workspace)
-		}
-		return fmt.Sprintf("semantic query is not ready yet for workspace %q\nThe vector backend will land in the next retrieval slice.", workspace)
-	default:
-		if !cfg.Query.Embeddings.Enabled {
-			return fmt.Sprintf("workspace query is not ready yet for workspace %q\nPlain query will use hybrid ranking and fall back to BM25 keywords when embeddings are disabled.\nUntil then, use '%s grep <pattern>' for exact line matches. Use '%s query --semantic <query>' when you specifically want vector-only search.", workspace, filepath.Base(os.Args[0]), filepath.Base(os.Args[0]))
-		}
-		return fmt.Sprintf("workspace query is not ready yet for workspace %q\nThe QMD-style hybrid query backend will land in the next retrieval slice.", workspace)
-	}
-}
-
 func workspaceQueryIsExplicitExpand(query string) bool {
 	return strings.HasPrefix(strings.ToLower(strings.TrimSpace(query)), "expand:")
 }
@@ -757,7 +732,10 @@ QMD-style hybrid + rerank workspace query.
 Plain text runs hybrid retrieval by default. Use --keyword for keyword-ranked
 retrieval only, or --semantic for vector-only semantic search.
 
-If embeddings are disabled, default query falls back to keyword ranked results.
+Default query currently falls back to keyword ranked results until hybrid
+vector/rerank is complete. Use --semantic for vector-only retrieval. Semantic
+embeddings are globally enabled and use OpenAI when OPENAI_API_KEY is set in
+the control-plane environment.
 Use grep when you know the exact text.
 
 Typed query documents: