Upload advanced_rag_app.py

advanced_rag_app.py

@@ -0,0 +1,471 @@
+# ==============================================================================
+# ADVANCED RAG WITH GPT, LANGCHAIN, AND RAGAS EVALUATION
+# ==============================================================================
+# Enhanced RAG application with quality metrics using RAGAS framework
+# Supports multiple PDF documents
+# ==============================================================================
+
+from langchain.retrievers import EnsembleRetriever
+from langchain_community.retrievers import BM25Retriever
+from langchain_community.cross_encoders import HuggingFaceCrossEncoder
+from langchain.retrievers.document_compressors import CrossEncoderReranker
+from sentence_transformers import CrossEncoder
+from langchain.retrievers import ContextualCompressionRetriever
+from langchain_community.document_loaders import PyPDFLoader
+from langchain.text_splitter import RecursiveCharacterTextSplitter
+from langchain_openai import OpenAIEmbeddings, ChatOpenAI
+from langchain_community.vectorstores import FAISS
+from langchain.schema import Document
+from langchain.prompts import PromptTemplate
+from langchain_core.output_parsers import StrOutputParser
+from langchain_core.runnables import RunnablePassthrough
+from datasets import Dataset
+from ragas import evaluate
+from ragas.metrics import (
+    faithfulness,
+    answer_relevancy,
+    context_precision,
+    context_recall,
+    answer_correctness,
+    answer_similarity
+)
+import gradio as gr
+import os
+import pandas as pd
+import json
+
+# ==============================================================================
+# GLOBAL VARIABLES
+# ==============================================================================
+rag_chain = None
+current_documents = []  # Changed to list for multiple documents
+openai_api_key = None
+retriever = None
+evaluation_data = []
+
+# ==============================================================================
+# HELPER FUNCTIONS
+# ==============================================================================
+
+def format_docs(docs):
+    """Format retrieved documents with source citations"""
+    out = []
+    for d in docs:
+        src = d.metadata.get("source", "unknown")
+        # Extract just the filename from the full path
+        src = os.path.basename(src)
+        page = d.metadata.get("page", d.metadata.get("page_number", "?"))
+        
+        try:
+            page_display = int(page) + 1 
+        except (ValueError, TypeError):
+            page_display = page
+            
+        out.append(f"[{src}:{page_display}] {d.page_content}")
+    return "\n\n".join(out)
+
+
+def validate_api_key(api_key):
+    """Validate that API key is provided"""
+    if not api_key or not api_key.strip():
+        return False
+    return True
+
+
+def process_documents(pdf_files, api_key):
+    """Process uploaded PDFs and create RAG chain"""
+    global rag_chain, current_documents, openai_api_key, retriever, evaluation_data
+    
+    chatbot_clear = None 
+    evaluation_data = []  # Reset evaluation data
+    
+    if not validate_api_key(api_key):
+        return "⚠️ Please provide a valid OpenAI API key.", chatbot_clear, ""
+    
+    if pdf_files is None or len(pdf_files) == 0:
+        return "⚠️ Please upload at least one PDF file.", chatbot_clear, ""
+    
+    try:
+        openai_api_key = api_key.strip()
+        os.environ["OPENAI_API_KEY"] = openai_api_key
+        
+        # Process all uploaded PDFs
+        all_docs = []
+        current_documents = []
+        total_pages = 0
+        
+        for pdf_file in pdf_files:
+            loader = PyPDFLoader(pdf_file.name)
+            docs = loader.load()
+            all_docs.extend(docs)
+            current_documents.append(os.path.basename(pdf_file.name))
+            total_pages += len(docs)
+        
+        # Split all documents
+        splitter = RecursiveCharacterTextSplitter(
+            separators=["\n\n", "\n", ". ", " ", ""],
+            chunk_size=1000,
+            chunk_overlap=100
+        )
+        chunked_docs = splitter.split_documents(all_docs)
+        
+        # Create embeddings and vector store
+        embeddings = OpenAIEmbeddings(
+            model="text-embedding-3-small",
+            openai_api_key=openai_api_key
+        )
+        
+        db = FAISS.from_documents(chunked_docs, embeddings)
+        
+        retriever_1 = db.as_retriever(search_type="similarity",search_kwargs={'k': 10})
+        
+        retriever_2 = BM25Retriever.from_documents(chunked_docs, search_kwargs={"k": 10})
+        
+        ensemble_retriever = EnsembleRetriever(retrievers=[retriever_1, retriever_2], weights=[0.7, 0.3])
+        
+        cross_encoder_model = HuggingFaceCrossEncoder(model_name="cross-encoder/ms-marco-MiniLM-L-12-v2")
+
+        reranker = CrossEncoderReranker(model=cross_encoder_model,top_n=10)
+        
+        reranking_retriever = ContextualCompressionRetriever(base_compressor=reranker,base_retriever=ensemble_retriever)
+        
+        retriever=reranking_retriever  
+              
+        # Create LLM and prompt
+        llm = ChatOpenAI(
+            model="gpt-5-mini",
+            temperature=0.2,
+            openai_api_key=openai_api_key
+        )
+        
+        prompt_template = """You are a professional research scientist involved in document data analysis.
+        Use the following context to answer the question using information provided by the documents.
+        Answer using ONLY these passages. Cite sources as [filename:page] after each claim.
+        Provide an answer in bullet points.
+        If you can't find it, say you don't know.
+
+Question:
+{question}
+
+Passages:
+{context}
+
+Answer:"""
+        
+        prompt = PromptTemplate(
+            input_variables=["context", "question"],
+            template=prompt_template,
+        )
+        
+        llm_chain = prompt | llm | StrOutputParser()
+        
+        rag_chain = (
+            {"context": reranking_retriever | format_docs, "question": RunnablePassthrough()} 
+            | llm_chain
+        )
+        
+        # Create status message with document list
+        doc_list = "\n".join([f"  • {doc}" for doc in current_documents])
+        status_msg = (
+            f"✅ Documents processed successfully!\n\n"
+            f"📄 **Documents loaded ({len(current_documents)}):**\n{doc_list}\n\n"
+            f"📊 Total pages: {total_pages}\n"
+            f"📦 Chunks created: {len(chunked_docs)}\n\n"
+            f"You can now ask questions and evaluate responses!"
+        )
+        
+        return status_msg, chatbot_clear, ""
+        
+    except Exception as e:
+        return f"❌ Error processing documents: {str(e)}", chatbot_clear, ""
+
+
+def chat_with_document(message, history):
+    """Handle chat interactions with the documents"""
+    global rag_chain, current_documents, retriever, evaluation_data
+    
+    history.append({"role": "user", "content": message})
+    
+    if rag_chain is None:
+        history.append({
+            "role": "assistant", 
+            "content": "⚠️ Please upload and process PDF documents first."
+        })
+        return history
+    
+    if not message.strip():
+        history.append({
+            "role": "assistant", 
+            "content": "⚠️ Please enter a question."
+        })
+        return history
+    
+    try:
+        # Retrieve contexts for RAGAS evaluation
+        retrieved_docs = retriever.invoke(message)
+        contexts = [doc.page_content for doc in retrieved_docs]
+        
+        # Get response from RAG chain
+        response = rag_chain.invoke(message)
+        
+        if isinstance(response, dict):
+            res_text = response.get("answer", response.get("result", str(response)))
+        else:
+            res_text = str(response)
+        
+        # Store data for RAGAS evaluation
+        evaluation_data.append({
+            "question": message,
+            "answer": res_text,
+            "contexts": contexts
+        })
+        
+        history.append({"role": "assistant", "content": res_text})
+        return history
+        
+    except Exception as e:
+        error_msg = f"❌ Error generating response: {str(e)}"
+        history.append({"role": "assistant", "content": error_msg})
+        return history
+
+
+def evaluate_rag_performance():
+    """Evaluate RAG performance using RAGAS metrics"""
+    global evaluation_data, openai_api_key
+    
+    if not evaluation_data:
+        return "⚠️ No evaluation data available. Please ask some questions first."
+    
+    try:
+        # Prepare dataset for RAGAS
+        dataset_dict = {
+            "question": [item["question"] for item in evaluation_data],
+            "answer": [item["answer"] for item in evaluation_data],
+            "contexts": [item["contexts"] for item in evaluation_data],
+        }
+        
+        dataset = Dataset.from_dict(dataset_dict)
+        
+        # Run RAGAS evaluation
+        # Using only metrics that don't require ground truth (reference answers)
+        result = evaluate(
+            dataset,
+            metrics=[
+                faithfulness,
+                answer_relevancy,
+            ],
+            llm=ChatOpenAI(model="gpt-4o-mini", openai_api_key=openai_api_key),
+            embeddings=OpenAIEmbeddings(openai_api_key=openai_api_key),
+        )
+        
+        # Convert to DataFrame for better display
+        df = result.to_pandas()
+        
+        # Calculate average scores from the result directly
+        metrics_summary = "## 📊 RAGAS Evaluation Results\n\n"
+        metrics_summary += "### Average Scores:\n"
+        
+        # Get metric scores safely
+        metric_cols = ['faithfulness', 'answer_relevancy']
+        metric_scores = {}
+        
+        for col in metric_cols:
+            if col in df.columns:
+                # Convert to numeric, handling any non-numeric values
+                numeric_values = pd.to_numeric(df[col], errors='coerce')
+                avg_score = numeric_values.mean()
+                if not pd.isna(avg_score):
+                    metric_scores[col] = avg_score
+                    metrics_summary += f"- **{col.replace('_', ' ').title()}**: {avg_score:.4f}\n"
+        
+        metrics_summary += "\n### Metric Explanations:\n"
+        metrics_summary += "- **Faithfulness** (0-1): Measures if the answer is factually consistent with the retrieved context. Higher scores mean the answer doesn't hallucinate or contradict the source.\n"
+        metrics_summary += "- **Answer Relevancy** (0-1): Measures how relevant the answer is to the question asked. Higher scores mean better alignment with the user's query.\n"
+        
+        
+        metrics_summary += "\n### Interpretation Guide:\n"
+        metrics_summary += "- **0.9 - 1.0**: Excellent performance\n"
+        metrics_summary += "- **0.7 - 0.9**: Good performance\n"
+        metrics_summary += "- **0.5 - 0.7**: Moderate performance (needs improvement)\n"
+        metrics_summary += "- **< 0.5**: Poor performance (requires significant optimization)\n"
+        
+        metrics_summary += f"\n### Total Questions Evaluated: {len(evaluation_data)}\n"
+        
+        # Add document info
+        if current_documents:
+            metrics_summary += f"\n### Documents in Index: {len(current_documents)}\n"
+        
+        return metrics_summary
+        
+    except Exception as e:
+        return f"❌ Error during evaluation: {str(e)}"
+
+
+def export_evaluation_data():
+    """Export evaluation data as JSON"""
+    global evaluation_data, current_documents
+    
+    if not evaluation_data:
+        return None
+    
+    try:
+        # Create a temporary file with metadata
+        output_data = {
+            "documents": current_documents,
+            "evaluation_data": evaluation_data,
+            "total_questions": len(evaluation_data)
+        }
+        
+        output_path = "ragas_evaluation_data.json"
+        with open(output_path, 'w') as f:
+            json.dump(output_data, f, indent=2)
+        return output_path
+    except Exception as e:
+        print(f"Error exporting data: {str(e)}")
+        return None
+
+
+def clear_chat():
+    """Clear the chat history and evaluation data"""
+    global evaluation_data
+    evaluation_data = []  # Reset evaluation data when clearing chat
+    return [], ""  # Return empty chatbot and empty eval_summary
+
+
+
+# ==============================================================================
+# GRADIO INTERFACE
+# ==============================================================================
+
+with gr.Blocks(title="RAG with RAGAS Evaluation", theme=gr.themes.Soft()) as demo:
+    
+    gr.Markdown(
+        """
+        # 📚 Multi-Document Q&A Analysis 
+        ### Advanced RAG System Powered by OpenAI GPT models, LangChain & RAGAS
+        
+        Upload multiple PDFs, ask questions across all documents, and evaluate your RAG system's performance with industry-standard metrics.
+        """
+    )
+    
+    with gr.Row():
+        with gr.Column(scale=1):
+            gr.Markdown(
+                """
+                ### 📋 How to Use
+                1. Enter your OpenAI API key
+                2. Upload one or more PDF documents
+                3. Process the documents
+                4. Ask questions in the chat
+                5. Click "Evaluate" to see performance metrics
+                
+                ---
+                
+                💡 **RAGAS Metrics**:
+                - Faithfulness: Factual accuracy
+                - Answer Relevancy: Question alignment
+                
+                📁 **Multi-Document Support**:
+                - Upload multiple PDFs at once
+                - Search across all documents
+                - Get citations with document names
+                """
+            )
+            
+            gr.Markdown("### 🔑 API Configuration")
+            api_key_input = gr.Textbox(
+                label="OpenAI API Key",
+                type="password",
+                placeholder="sk-...",
+                info="Required for GPT models and RAGAS evaluation"
+            )
+            
+            gr.Markdown("### 📤 Upload Documents")
+            pdf_input = gr.File(
+                label="Upload PDF Documents",
+                file_types=[".pdf"],
+                type="filepath",
+                file_count="multiple"  # Enable multiple file upload
+            )
+            process_btn = gr.Button("📄 Process Documents", variant="primary", size="lg")
+            
+            status_output = gr.Textbox(
+                label="Status",
+                lines=8,  # Increased to show multiple documents
+                interactive=False,
+                placeholder="Enter API key, upload PDFs, and click 'Process Documents'..."
+            )
+            
+            gr.Markdown("### 📈 Evaluation")
+            evaluate_btn = gr.Button("🔍 Evaluate RAG Performance", variant="secondary", size="lg")
+            export_btn = gr.Button("💾 Export Evaluation Data", size="sm")
+            export_file = gr.File(label="Download Evaluation Data", visible=True)
+        
+        with gr.Column(scale=2):
+            gr.Markdown("### 💬 Chat with Your Documents")
+            chatbot = gr.Chatbot(
+                height=400,
+                placeholder="Upload and process documents to start...",
+                show_label=False,
+                type="messages" 
+            )
+            
+            msg = gr.Textbox(
+                label="Enter your question",
+                placeholder="Type your question here (searches across all uploaded documents)...",
+                lines=2
+            )
+            
+            with gr.Row():
+                submit_btn = gr.Button("📤 Send", variant="primary", scale=4)
+                clear_btn = gr.Button("🗑️ Clear Chat", scale=1)
+            
+            gr.Markdown("### 📊 Evaluation Results")
+            eval_summary = gr.Markdown(value="")
+    
+    # Event handlers
+    process_btn.click(
+        fn=process_documents,  # Changed function name
+        inputs=[pdf_input, api_key_input],
+        outputs=[status_output, chatbot, eval_summary] 
+    )
+    
+    submit_btn.click(
+        fn=chat_with_document,
+        inputs=[msg, chatbot], 
+        outputs=[chatbot] 
+    ).then(
+        lambda: "", 
+        outputs=[msg] 
+    )
+    
+    msg.submit(
+        fn=chat_with_document,
+        inputs=[msg, chatbot], 
+        outputs=[chatbot] 
+    ).then(
+        lambda: "", 
+        outputs=[msg] 
+    )
+    
+    clear_btn.click(
+        fn=clear_chat,
+        outputs=[chatbot, eval_summary]
+    )
+    
+    evaluate_btn.click(
+        fn=evaluate_rag_performance,
+        outputs=[eval_summary]
+    )
+    
+    export_btn.click(
+        fn=export_evaluation_data,
+        outputs=[export_file]
+    )
+
+# ==============================================================================
+# LAUNCH APPLICATION
+# ==============================================================================
+
+if __name__ == "__main__":
+    demo.launch(share=False, debug=True)