feat: improve search results and file handling

- Improve search results formatting with better indentation and code blocks
- Change 'filename' to 'source' in metadata for consistency
- Enhance gitignore pattern handling with more default patterns
- Remove redundant ignore patterns from watch command (now using gitignore)
- Use code blocks with source as language in context assembly

Author: ErikBjare

gptme_rag/cli.py

@@ -97,31 +97,31 @@ def search(
         # Show a summary of the most relevant documents
         console.print("\n[bold]Most Relevant Documents:[/bold]")
         for i, doc in enumerate(documents):
-            filename = doc.metadata.get("filename", "unknown")
+            source = doc.metadata.get("source", "unknown")
             distance = distances[i]
             relevance = 1 - distance  # Convert distance to similarity score
 
             # Show document header with relevance score
             console.print(
-                f"\n[cyan]{i+1}. {filename}[/cyan] [yellow](relevance: {relevance:.2f})[/yellow]"
+                f"\n[cyan]{i+1}. {source}[/cyan] [yellow](relevance: {relevance:.2f})[/yellow]"
             )
 
             # Extract first meaningful section (after headers)
             lines = doc.content.split("\n")
             content = []
             for line in lines:
                 if line.strip() and not line.startswith("#"):
-                    content.append(line.strip())
-                    if len(" ".join(content)) > 200:
+                    content.append("  " + line + "\n")
+                    if len("".join(content)) > 200:
                         break
 
             # Show the first paragraph or meaningful content
             content_preview = (
-                " ".join(content)[:200] + "..."
-                if len(" ".join(content)) > 200
-                else " ".join(content)
+                "".join(content)[:200] + "..."
+                if len("".join(content)) > 200
+                else "".join(content)
             )
-            console.print(f"  {content_preview}")
+            console.print(f"{content_preview}")
 
         # Assemble context window
         context = assembler.assemble_context(documents, user_query=query)
@@ -158,7 +158,7 @@ def search(
     "--ignore-patterns",
     "-i",
     multiple=True,
-    default=[".git", "__pycache__", "*.pyc"],
+    default=[],
     help="Glob patterns to ignore",
 )
 def watch(directory: Path, pattern: str, persist_dir: Path, ignore_patterns: list[str]):
@@ -174,6 +174,7 @@ def watch(directory: Path, pattern: str, persist_dir: Path, ignore_patterns: lis
         console.print("Starting file watcher...")
 
         try:
+            # TODO: FileWatcher should use same gitignore patterns as indexer
             file_watcher = FileWatcher(
                 indexer, [str(directory)], pattern, ignore_patterns
             )

gptme_rag/indexing/indexer.py

@@ -172,7 +172,17 @@ def add_documents(self, documents: list[Document], batch_size: int = 100) -> Non
 
     def _load_gitignore(self, directory: Path) -> list[str]:
         """Load gitignore patterns from all .gitignore files up to root."""
-        patterns: list[str] = [".git/", ".sqlite3", ".db"]
+        # arguably only .git/** should be here, with the rest in system global gitignore (which should be respected)
+        patterns: list[str] = [
+            ".git/**",
+            "*.sqlite3",
+            "*.db",
+            "*.pyc",
+            "__pycache__",
+            ".*cache/**",
+            "*.lock",
+            ".DS_Store",
+        ]
         current_dir = directory.resolve()
         max_depth = 10  # Limit traversal to avoid infinite loops
 
@@ -201,7 +211,11 @@ def _is_ignored(self, file_path: Path, gitignore_patterns: list[str]) -> bool:
         rel_path = str(file_path)
 
         for pattern in gitignore_patterns:
-            if fnmatch(rel_path, pattern) or fnmatch(rel_path, f"**/{pattern}"):
+            if (
+                fnmatch(rel_path, pattern)
+                or fnmatch(rel_path, f"**/{pattern}")
+                or fnmatch(rel_path, f"**/{pattern}/**")
+            ):
                 return True
         return False
 

gptme_rag/query/context_assembler.py

@@ -28,7 +28,7 @@ def _count_tokens(self, text: str) -> int:
     def _format_document(self, doc: Document) -> str:
         """Format a document for inclusion in the context window."""
         source = doc.metadata.get("source", "unknown")
-        return f"--- From {source} ---\n{doc.content}\n"
+        return f"```{source}\n{doc.content}\n```"
 
     def assemble_context(
         self,