fix: dont include duplicate chunks in context assembly (fixes #7)

Author: ErikBjare

gptme_rag/query/context_assembler.py

@@ -1,9 +1,12 @@
+import logging
 from dataclasses import dataclass
 
 import tiktoken
 
 from ..indexing.document import Document
 
+logger = logging.getLogger(__name__)
+
 
 @dataclass
 class ContextWindow:
@@ -40,9 +43,10 @@ def assemble_context(
         Assemble a context window from documents, staying within token limit.
 
         Documents should be pre-sorted by relevance.
+        Duplicate documents will be filtered out and a warning will be logged.
         """
         total_tokens = 0
-        included_docs = []
+        included_docs: list[Document] = []
         context_parts = []
         truncated = False
 
@@ -64,6 +68,11 @@ def assemble_context(
             formatted_doc = self._format_document(doc)
             doc_tokens = self._count_tokens(formatted_doc)
 
+            # check if document content is duplicate
+            if doc.content in [d.content for d in included_docs]:
+                logger.warning(f"Duplicate document found: {doc.metadata['source']}")
+                continue
+
             if total_tokens + doc_tokens > self.max_tokens:
                 truncated = True
                 break