Introduce `RagPipeline` for conversational RAG, including vector store management and query processing.

core/rag_pipeline.py

@@ -1,11 +1,13 @@
 import logging
 import time
+import os
 from langchain_core.messages import HumanMessage, AIMessage
 from langchain.chains import create_history_aware_retriever, create_retrieval_chain
 from langchain.chains.combine_documents import create_stuff_documents_chain
 from config.settings import Settings
 from services.llm_client import LLMClient
 from services.vector_store import VectorStore
+from services.document_processor import DocumentProcessor
 from core.prompts import get_chat_prompt, get_contextualize_prompt
 
 logger = logging.getLogger(__name__)
@@ -15,6 +17,28 @@ class RagPipeline:
         try:
             self.llm = LLMClient().get_llm()
             self.vector_store = VectorStore()
+
+            try:
+                logger.info("Checking Vector Database integrity...")
+                test_retriever = self.vector_store.get_retriever(k=1)
+                test_retriever.invoke("test") 
+                logger.info("Vector Database is healthy and ready.")
+                
+            except Exception as e:
+                logger.warning(f"Database seems empty or corrupt ({str(e)}). Rebuilding from PDF...")
+                
+                try:
+                    processor = DocumentProcessor()
+                    docs = processor.process_documents("./data")
+                    
+                    if not docs:
+                        logger.error("No documents found in ./data folder to ingest!")
+                    else:
+                        self.vector_store.add_documents(docs)
+                        logger.info(f"Successfully rebuilt database with {len(docs)} chunks.")
+                except Exception as build_error:
+                    logger.error(f"Failed to rebuild database: {str(build_error)}")
+            
             self.retriever = self.vector_store.get_retriever(k=5)
             self.prompt = get_chat_prompt()
             self.history_aware_retriever = create_history_aware_retriever(
@@ -30,7 +54,6 @@ class RagPipeline:
                 self.history_aware_retriever,
                 self.question_answer_chain
             )
-            self.chat_history = []
             logger.info("RAG pipeline initialized successfully")
 
         except Exception as e:
@@ -38,22 +61,23 @@ class RagPipeline:
             raise e
 
     def clear_history(self):
-        self.chat_history = []
-        logger.info("Chat history cleared")
+        pass
 
     def process_query(self, question:str, chat_history: list = []):
         start_time = time.time()
         try:
             logger.info(f"Processing query: {question}")
+            langchain_history = []
+            for msg in chat_history:
+                if msg[0] == "human":
+                    langchain_history.append(HumanMessage(content=msg[1]))
+                elif msg[0] == "ai":
+                    langchain_history.append(AIMessage(content=msg[1]))
+            
             response = self.rag_chain.invoke({
                 "input": question,
-                "chat_history": self.chat_history
+                "chat_history": langchain_history
                 })
-            
-            self.chat_history.extend([
-                HumanMessage(content=question),
-                AIMessage(content=response["answer"])
-            ])
 
             latency = time.time() - start_time