perf: only index new/modified files

Track file modification times and skip unchanged files during indexing.
This improves performance by avoiding unnecessary re-indexing.

Key changes:
- Track file modification times in document metadata
- Compare mtimes to detect modified files
- Only process new or modified files
- Improve logging and status messages

Author: ErikBjare

gptme_rag/cli.py

@@ -57,26 +57,68 @@ def index(paths: list[Path], pattern: str, persist_dir: Path):
     try:
         indexer = Indexer(persist_directory=persist_dir, enable_persist=True)
 
-        # First, collect all documents
+        # Get existing files and their metadata from the index, using absolute paths
+        existing_docs = indexer.get_all_documents()
+        logger.debug("Found %d existing documents in index", len(existing_docs))
+
+        existing_files = {}
+        for doc in existing_docs:
+            if "source" in doc.metadata:
+                abs_path = os.path.abspath(doc.metadata["source"])
+                mtime = doc.metadata.get("mtime", 0)
+                existing_files[abs_path] = mtime
+                logger.debug("Existing file: %s (mtime: %s)", abs_path, mtime)
+
+        logger.debug("Loaded %d existing files from index", len(existing_files))
+
+        # First, collect all documents and filter for new/modified
         all_documents = []
         with console.status("Collecting documents...") as status:
             for path in paths:
                 if path.is_file():
                     status.update(f"Processing file: {path}")
                 else:
                     status.update(f"Processing directory: {path}")
+
                 documents = indexer.collect_documents(path)
-                all_documents.extend(documents)
+
+                # Filter for new or modified documents
+                filtered_documents = []
+                for doc in documents:
+                    source = doc.metadata.get("source")
+                    if source:
+                        # Resolve to absolute path for consistent comparison
+                        abs_source = os.path.abspath(source)
+                        doc.metadata["source"] = abs_source
+                        current_mtime = os.path.getmtime(abs_source)
+                        doc.metadata["mtime"] = current_mtime
+
+                        # Include if file is new or modified
+                        if abs_source not in existing_files:
+                            logger.debug("New file: %s", abs_source)
+                            filtered_documents.append(doc)
+                        elif current_mtime > existing_files[abs_source]:
+                            logger.debug(
+                                "Modified file: %s (current: %s, stored: %s)",
+                                abs_source,
+                                current_mtime,
+                                existing_files[abs_source],
+                            )
+                            filtered_documents.append(doc)
+                        else:
+                            logger.debug("Unchanged file: %s", abs_source)
+
+                all_documents.extend(filtered_documents)
 
         if not all_documents:
-            console.print("No documents found to index", style="yellow")
+            console.print("No new or modified documents to index", style="yellow")
             return
 
         # Then process them with a progress bar
         n_files = len(set(doc.metadata.get("source", "") for doc in all_documents))
         n_chunks = len(all_documents)
 
-        logger.info(f"Found {n_files} files to index ({n_chunks} chunks)")
+        logger.info(f"Found {n_files} new/modified files to index ({n_chunks} chunks)")
 
         with tqdm(
             total=n_chunks,

gptme_rag/indexing/indexer.py

@@ -905,3 +905,11 @@ def index_file(self, path: Path) -> int:
             self.add_documents(documents)
             return len(documents)
         return 0
+
+    def get_all_documents(self) -> list[Document]:
+        """Get all documents from the index.
+
+        Returns:
+            List of all documents in the index, including all chunks.
+        """
+        return self.list_documents(group_by_source=False)