/**
 * useMultiDatabaseRAG.ts - Multi-Database RAG (Retrieval-Augmented Generation) Hook
 * 
 * Orchestrates queries across multiple databases for conversational AI:
 * - Qdrant: Vector similarity search for semantic retrieval
 * - Oxigraph: SPARQL queries for structured RDF data
 * - TypeDB: TypeQL queries for knowledge graph traversal
 * 
 * Features intelligent semantic caching to reduce API costs:
 * - Caches query embeddings and responses in IndexedDB
 * - Uses cosine similarity to find semantically similar past queries
 * - Configurable similarity threshold (default 0.92)
 * - Typical API savings: 30-50% for conversational UIs
 * 
 * Based on DSPy RAG patterns for heritage institution conversations.
 * Self-hosted infrastructure - no external API keys required.
 * 
 * @see https://dspy.ai/
 */

import { useState, useCallback, useRef, useEffect } from 'react';
import type { QdrantSearchResult } from './useQdrant';
import { semanticCache, type CachedResponse, type CacheStats, type CacheLookupResult } from '../lib/storage/semantic-cache';

// Configuration - all services use Caddy proxy paths
const API_BASE = '';  // Relative URLs via Caddy proxy
const QDRANT_URL = '/qdrant';
const SPARQL_URL = '/sparql';
const TYPEDB_URL = '/api/typedb';
const DSPY_URL = '/api/dspy/rag';  // Backend serves at /api/dspy/rag/query

// ============================================================================
// Types
// ============================================================================

export interface RAGContext {
  qdrantResults: QdrantSearchResult[];
  sparqlResults: Record<string, unknown>[];
  typedbResults: Record<string, unknown>[];
  totalRetrieved: number;
}

/**
 * Retrieved result from backend - can be a person or institution
 * Returned by HeritageRAGPipeline.forward() for visualization
 */
export interface RetrievedResult {
  type: 'person' | 'institution';
  // Person fields (when type === 'person')
  person_id?: string;
  name: string;
  headline?: string;
  custodian_name?: string;
  custodian_slug?: string;
  heritage_relevant?: boolean;
  heritage_type?: string;  // GLAMORCUBESFIXPHDNT single letter code
  linkedin_url?: string | null;
  score?: number;
  // Institution fields (when type === 'institution')
  institution_type?: string;
  city?: string;
  country?: string;
  description?: string;
}

export type QueryType = 'person' | 'institution';

export interface RAGResponse {
  answer: string;
  sparqlQuery?: string;
  typeqlQuery?: string;
  context: RAGContext;
  visualizationType?: VisualizationType;
  visualizationData?: VisualizationData;
  sources: RAGSource[];
  confidence: number;
  // New fields from backend for person/institution visualization
  retrievedResults?: RetrievedResult[];
  queryType?: QueryType;
}

export interface RAGSource {
  database: 'qdrant' | 'oxigraph' | 'typedb';
  id: string;
  name?: string;
  score?: number;
  snippet?: string;
}

export type VisualizationType = 
  | 'none'
  | 'map'           // Geographic visualization
  | 'timeline'      // Temporal visualization
  | 'network'       // Graph/relationship visualization
  | 'chart'         // Bar/line charts
  | 'table'         // Tabular data
  | 'card'          // Institution cards
  | 'gallery';      // Image gallery

export interface VisualizationData {
  type: VisualizationType;
  institutions?: InstitutionData[];
  coordinates?: GeoCoordinate[];
  timeline?: TimelineEvent[];
  graphData?: GraphVisualizationData;
  chartData?: ChartData;
}

export interface InstitutionData {
  id: string;
  name: string;
  type?: string;
  city?: string;
  province?: string;
  country?: string;
  latitude?: number;
  longitude?: number;
  description?: string;
  website?: string;
  isil?: string;
  wikidata?: string;
  rating?: number;
  reviews?: number;
  photoCount?: number;
}

export interface GeoCoordinate {
  lat: number;
  lng: number;
  label: string;
  type?: string;
  data?: InstitutionData;
}

export interface TimelineEvent {
  date: string;
  label: string;
  description?: string;
  type?: string;
}

export interface GraphVisualizationData {
  nodes: Array<{
    id: string;
    label: string;
    type: string;
    attributes?: Record<string, unknown>;
  }>;
  edges: Array<{
    id: string;
    source: string;
    target: string;
    label: string;
    type?: string;
  }>;
}

export interface ChartData {
  labels: string[];
  datasets: Array<{
    label: string;
    data: number[];
    backgroundColor?: string | string[];
    borderColor?: string;
  }>;
}

export interface ConversationMessage {
  id: string;
  role: 'user' | 'assistant' | 'system';
  content: string;
  timestamp: Date;
  response?: RAGResponse;
  isLoading?: boolean;
  error?: string;
}

export interface UseMultiDatabaseRAGReturn {
  // State
  isLoading: boolean;
  error: Error | null;
  lastContext: RAGContext | null;
  
  // Cache state
  cacheEnabled: boolean;
  lastCacheLookup: CacheLookupResult | null;
  
  // Core RAG function
  queryRAG: (
    question: string,
    options?: RAGOptions
  ) => Promise<RAGResponse>;
  
  // Individual database queries (for debugging/advanced use)
  searchQdrant: (query: string, limit?: number) => Promise<QdrantSearchResult[]>;
  querySparql: (sparql: string) => Promise<Record<string, unknown>[]>;
  queryTypeDB: (typeql: string) => Promise<Record<string, unknown>[]>;
  
  // Utility functions
  clearContext: () => void;
  detectVisualizationType: (question: string, results: RAGContext) => VisualizationType;
  
  // Cache management functions
  setCacheEnabled: (enabled: boolean) => void;
  getCacheStats: () => Promise<CacheStats>;
  clearCache: () => Promise<{ localCleared: boolean; sharedCleared: boolean }>;
  setCacheSimilarityThreshold: (threshold: number) => void;
}

export interface RAGOptions {
  model?: string;
  language?: 'nl' | 'en';
  maxQdrantResults?: number;
  maxSparqlResults?: number;
  maxTypeDBResults?: number;
  includeSparql?: boolean;
  includeTypeDB?: boolean;
  conversationHistory?: ConversationMessage[];
  // Cache options
  useCache?: boolean;           // Enable/disable cache for this query (default: true)
  bypassCache?: boolean;        // Force fresh query even if cache hit (default: false)
  storeInCache?: boolean;       // Store result in cache (default: true)
  // Embedding model selection
  embeddingModel?: 'minilm_384' | 'openai_1536' | 'bge_768' | null;  // Embedding model for vector search (default: auto)
}

// ============================================================================
// Helper Functions
// ============================================================================

/**
 * Generate text embedding using local embedding service or fallback
 * In production, this would use a local embedding model (e.g., sentence-transformers)
 * For now, we'll use keyword-based Qdrant filtering as a fallback
 */
async function generateEmbedding(text: string): Promise<number[] | null> {
  try {
    // Try local embedding service first
    const response = await fetch(`${API_BASE}/api/embed`, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({ text }),
    });
    
    if (response.ok) {
      const data = await response.json();
      return data.embedding;
    }
  } catch {
    // Fallback: return null to use keyword search
  }
  return null;
}

/**
 * Search Qdrant using vector similarity or keyword filter
 */
async function qdrantSearch(
  query: string,
  limit: number = 10
): Promise<QdrantSearchResult[]> {
  const collectionName = 'heritage_custodians';
  
  console.log('[Qdrant] Search query:', query);
  console.log('[Qdrant] Limit:', limit);
  console.log('[Qdrant] QDRANT_URL:', QDRANT_URL);
  
  // Try to get embedding for semantic search
  const embedding = await generateEmbedding(query);
  console.log('[Qdrant] Has embedding:', !!embedding);
  
  if (embedding) {
    // Vector similarity search
    const searchUrl = `${QDRANT_URL}/collections/${collectionName}/points/search`;
    console.log('[Qdrant] Search URL:', searchUrl);
    console.log('[Qdrant] Full search URL:', new URL(searchUrl, window.location.origin).href);
    
    const response = await fetch(searchUrl, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify({
        vector: embedding,
        limit,
        with_payload: true,
      }),
    });
    
    console.log('[Qdrant] Search response status:', response.status);
    if (!response.ok) {
      const errorBody = await response.text();
      console.error('[Qdrant] Search error body:', errorBody);
    }
    
    if (response.ok) {
      const data = await response.json();
      console.log('[Qdrant] Search results count:', data.result?.length || 0);
      return data.result || [];
    }
  }
  
  // Fallback: Scroll through points with keyword filter
  // Extract keywords from query for filtering
  const keywords = query.toLowerCase().split(/\s+/).filter(w => w.length > 2);
  console.log('[Qdrant] Fallback: Keywords:', keywords);
  
  const scrollUrl = `${QDRANT_URL}/collections/${collectionName}/points/scroll`;
  console.log('[Qdrant] Scroll URL:', scrollUrl);
  console.log('[Qdrant] Full scroll URL:', new URL(scrollUrl, window.location.origin).href);
  
  const response = await fetch(scrollUrl, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({
      limit: limit * 2,  // Get more to filter
      with_payload: true,
      with_vector: false,
    }),
  });
  
  console.log('[Qdrant] Scroll response status:', response.status);
  
  if (!response.ok) {
    const errorBody = await response.text();
    console.error('[Qdrant] Scroll error body:', errorBody);
    throw new Error(`Qdrant scroll failed: ${response.status}`);
  }
  
  const data = await response.json();
  const points = data.result?.points || [];
  
  // Simple keyword matching in payload
  const scored = points.map((p: { id: string | number; payload: Record<string, unknown> }) => {
    const payload = p.payload || {};
    const text = JSON.stringify(payload).toLowerCase();
    const matches = keywords.filter(k => text.includes(k)).length;
    return {
      id: p.id,
      score: matches / Math.max(keywords.length, 1),
      payload,
    };
  });
  
  // Sort by score and return top results
  return scored
    .filter((p: { score: number }) => p.score > 0)
    .sort((a: { score: number }, b: { score: number }) => b.score - a.score)
    .slice(0, limit);
}

/**
 * Execute SPARQL query against Oxigraph
 */
async function sparqlQuery(query: string): Promise<Record<string, unknown>[]> {
  const response = await fetch(`${SPARQL_URL}/query`, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/sparql-query',
      'Accept': 'application/sparql-results+json',
    },
    body: query,
  });
  
  if (!response.ok) {
    const error = await response.text();
    throw new Error(`SPARQL query failed: ${response.status} - ${error}`);
  }
  
  const data = await response.json();
  return data.results?.bindings || [];
}

/**
 * Execute TypeQL query against TypeDB
 */
async function typedbQuery(query: string): Promise<Record<string, unknown>[]> {
  const response = await fetch(`${TYPEDB_URL}/query`, {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ query, queryType: 'read' }),
  });
  
  if (!response.ok) {
    const error = await response.text();
    throw new Error(`TypeDB query failed: ${response.status} - ${error}`);
  }
  
  const data = await response.json();
  return data.results || [];
}

/**
 * Call DSPy backend to generate queries and response
 */
async function callDSPy(
  question: string,
  context: RAGContext,
  options: RAGOptions
): Promise<{
  answer: string;
  sparqlQuery?: string;
  typeqlQuery?: string;
  visualizationType?: VisualizationType;
  confidence: number;
  retrievedResults?: RetrievedResult[];
  queryType?: QueryType;
}> {
  // Format conversation history for DSPy backend
  // Backend expects: context = [{question: "...", answer: "..."}, ...]
  // Each object should pair a user question with its corresponding assistant answer
  const conversationContext: Array<{ question: string; answer: string }> = [];
  const messages = options.conversationHistory || [];
  
  // Pair consecutive user/assistant messages (skip last if current user question)
  for (let i = 0; i < messages.length - 1; i++) {
    const current = messages[i];
    const next = messages[i + 1];
    
    // If we have a user message followed by an assistant message, pair them
    if (current.role === 'user' && next.role === 'assistant' && !next.isLoading) {
      conversationContext.push({
        question: current.content,
        answer: next.content,
      });
      i++; // Skip the assistant message since we've paired it
    }
  }
  
  // Keep only last 4 turns for context
  const recentContext = conversationContext.slice(-4);
  
  const requestUrl = `${DSPY_URL}/query`;
  const requestBody = {
    question,
    language: options.language || 'nl',
    context: recentContext,  // Backend expects conversation history here (paired Q&A)
    include_visualization: true,
    embedding_model: options.embeddingModel || null,  // Pass embedding model preference
  };
  
  console.log('[DSPy] Request URL:', requestUrl);
  console.log('[DSPy] Request body:', JSON.stringify(requestBody, null, 2));
  console.log('[DSPy] Window location:', window.location.href);
  console.log('[DSPy] Full URL being fetched:', new URL(requestUrl, window.location.origin).href);
  
  let response: Response;
  try {
    response = await fetch(requestUrl, {
      method: 'POST',
      headers: { 'Content-Type': 'application/json' },
      body: JSON.stringify(requestBody),
    });
    console.log('[DSPy] Response status:', response.status);
    console.log('[DSPy] Response URL:', response.url);
    console.log('[DSPy] Response headers:', Object.fromEntries(response.headers.entries()));
  } catch (networkError) {
    // Network error - server unreachable
    console.error('[DSPy] Network error:', networkError);
    const lang = options.language || 'nl';
    return {
      answer: lang === 'nl'
        ? '⚠️ **Serverfout**: Kan geen verbinding maken met de RAG-server. Controleer of de backend draait op poort 8003.'
        : '⚠️ **Server Error**: Cannot connect to RAG server. Check if backend is running on port 8003.',
      confidence: 0,
    };
  }
  
  if (!response.ok) {
    // HTTP error - log details for debugging
    const responseBody = await response.text();
    console.error(`[DSPy] HTTP ${response.status}: ${response.statusText}`);
    console.error(`[DSPy] Response body:`, responseBody);
    const lang = options.language || 'nl';
    
    if (response.status === 404) {
      return {
        answer: lang === 'nl'
          ? '⚠️ **Serverfout (404)**: De RAG API endpoint is niet gevonden. Controleer de proxy configuratie in vite.config.ts en herstart de frontend.'
          : '⚠️ **Server Error (404)**: RAG API endpoint not found. Check proxy configuration in vite.config.ts and restart frontend.',
        confidence: 0,
      };
    }
    
    if (response.status >= 500) {
      return {
        answer: lang === 'nl'
          ? `⚠️ **Serverfout (${response.status})**: De RAG-server heeft een interne fout. Controleer de backend logs.`
          : `⚠️ **Server Error (${response.status})**: RAG server internal error. Check backend logs.`,
        confidence: 0,
      };
    }
    
    // Other HTTP errors - fall back to context-based answer
    return {
      answer: generateFallbackAnswer(question, context, lang),
      confidence: 0.5,
    };
  }
  
  // Map backend DSPyQueryResponse to frontend expected format
  const data = await response.json();
  console.log('[DSPy] Response data (first 500 chars):', JSON.stringify(data, null, 2).slice(0, 500));
  console.log('[DSPy] retrieved_results:', data.retrieved_results);
  console.log('[DSPy] retrieved_results count:', data.retrieved_results?.length ?? 0);
  console.log('[DSPy] query_type:', data.query_type);
  
  // Debug: log first result if available
  if (data.retrieved_results?.length > 0) {
    console.log('[DSPy] First retrieved result:', JSON.stringify(data.retrieved_results[0], null, 2));
  }
  
  // Flatten nested metadata structure from backend to match frontend RetrievedResult interface
  // Backend returns: { person_id, name, scores: {...}, metadata: { headline, custodian_name, ... }, type }
  // Frontend expects: { person_id, name, headline, custodian_name, ..., type }
  const flattenedResults: RetrievedResult[] | undefined = data.retrieved_results?.map((result: Record<string, unknown>) => {
    const metadata = result.metadata as Record<string, unknown> | undefined;
    const scores = result.scores as Record<string, number> | undefined;
    
    // If already flat (no metadata object), still normalize to ensure consistent structure
    if (!metadata) {
      return {
        type: (result.type as 'person' | 'institution') || 'institution',
        person_id: result.person_id as string | undefined,
        name: (result.name as string) || 'Unknown',
        headline: result.headline as string | undefined,
        custodian_name: result.custodian_name as string | undefined,
        custodian_slug: result.custodian_slug as string | undefined,
        heritage_relevant: result.heritage_relevant as boolean | undefined,
        heritage_type: result.heritage_type as string | undefined,
        linkedin_url: result.linkedin_url as string | undefined,
        score: result.score as number | undefined,
        institution_type: result.institution_type as string | undefined,
        city: result.city as string | undefined,
        country: result.country as string | undefined,
        description: result.description as string | undefined,
      };
    }
    
    // Flatten metadata into top-level fields
    return {
      type: (result.type as 'person' | 'institution') || 'institution',
      person_id: result.person_id as string | undefined,
      name: (result.name as string) || 'Unknown',
      headline: metadata.headline as string | undefined,
      custodian_name: metadata.custodian_name as string | undefined,
      custodian_slug: metadata.custodian_slug as string | undefined,
      location: metadata.location as string | undefined,
      heritage_relevant: metadata.heritage_relevant as boolean | undefined,
      heritage_type: metadata.heritage_type as string | undefined,
      linkedin_url: metadata.linkedin_url as string | undefined,
      score: scores?.combined || scores?.vector,
      // Institution fields (when type === 'institution')
      institution_type: metadata.institution_type as string | undefined,
      city: metadata.city as string | undefined,
      country: metadata.country as string | undefined,
      description: metadata.description as string | undefined,
    };
  });
  
  console.log('[DSPy] Flattened results count:', flattenedResults?.length ?? 0);
  if (flattenedResults?.length) {
    console.log('[DSPy] First flattened result:', JSON.stringify(flattenedResults[0], null, 2));
  }
  
  return {
    answer: data.answer || '',
    sparqlQuery: data.visualization?.sparql_query,  // If backend includes SPARQL
    visualizationType: data.visualization?.type as VisualizationType,
    confidence: data.sources_used?.length > 0 ? 0.85 : 0.6,  // Estimate confidence
    // New fields from backend for person/institution visualization
    retrievedResults: flattenedResults,
    queryType: data.query_type as QueryType | undefined,
  };
}

/**
 * Generate a fallback answer when DSPy service is unavailable
 */
function generateFallbackAnswer(
  _question: string,
  context: RAGContext,
  language: 'nl' | 'en'
): string {
  const count = context.totalRetrieved;
  
  if (count === 0) {
    return language === 'nl' 
      ? 'Geen resultaten gevonden voor uw vraag.'
      : 'No results found for your question.';
  }
  
  const institutions = context.qdrantResults.slice(0, 5).map(r => {
    const name = r.payload?.name || r.payload?.custodian_name || 'Unknown';
    return name;
  });
  
  if (language === 'nl') {
    return `Ik heb ${count} resultaten gevonden. Enkele relevante instellingen: ${institutions.join(', ')}.`;
  }
  return `I found ${count} results. Some relevant institutions: ${institutions.join(', ')}.`;
}

/**
 * Detect appropriate visualization type based on question and results
 */
function detectVisualizationType(
  question: string,
  context: RAGContext
): VisualizationType {
  const q = question.toLowerCase();
  
  // Map visualization keywords
  if (q.includes('kaart') || q.includes('map') || q.includes('waar') || 
      q.includes('where') || q.includes('locatie') || q.includes('location') ||
      q.includes('provincie') || q.includes('province') || q.includes('stad') ||
      q.includes('city') || q.includes('geografisch') || q.includes('geographic')) {
    return 'map';
  }
  
  // Timeline keywords
  if (q.includes('wanneer') || q.includes('when') || q.includes('geschiedenis') ||
      q.includes('history') || q.includes('tijdlijn') || q.includes('timeline') ||
      q.includes('opgericht') || q.includes('founded') || q.includes('jaar') ||
      q.includes('year')) {
    return 'timeline';
  }
  
  // Network/graph keywords
  if (q.includes('relatie') || q.includes('relationship') || q.includes('verbinding') ||
      q.includes('connection') || q.includes('netwerk') || q.includes('network') ||
      q.includes('samenwer') || q.includes('collaborat')) {
    return 'network';
  }
  
  // Chart keywords
  if (q.includes('hoeveel') || q.includes('how many') || q.includes('aantal') ||
      q.includes('count') || q.includes('statistiek') || q.includes('statistic') ||
      q.includes('verdeling') || q.includes('distribution') || q.includes('vergelijk') ||
      q.includes('compare')) {
    return 'chart';
  }
  
  // If we have location data, show map
  const hasCoordinates = context.qdrantResults.some(r => 
    r.payload?.latitude || r.payload?.coordinates
  );
  if (hasCoordinates && context.totalRetrieved > 0) {
    return 'map';
  }
  
  // Default to cards for institution results
  if (context.qdrantResults.length > 0) {
    return 'card';
  }
  
  return 'table';
}

/**
 * Extract visualization data from RAG context
 */
function extractVisualizationData(
  type: VisualizationType,
  context: RAGContext
): VisualizationData {
  const data: VisualizationData = { type };
  
  // Extract institution data from Qdrant results
  data.institutions = context.qdrantResults.map(r => {
    const p = (r.payload || {}) as Record<string, unknown>;
    const location = (p.location || {}) as Record<string, unknown>;
    const coordinates = (p.coordinates || {}) as Record<string, unknown>;
    return {
      id: String(r.id),
      name: String(p.name || p.custodian_name || p.institution_name || 'Unknown'),
      type: String(p.type || p.institution_type || ''),
      city: String(p.city || location.city || ''),
      province: String(p.province || p.region || ''),
      country: String(p.country || 'NL'),
      latitude: Number(p.latitude || coordinates.lat || location.latitude),
      longitude: Number(p.longitude || coordinates.lng || location.longitude),
      description: String(p.description || ''),
      website: String(p.website || p.url || ''),
      isil: String(p.isil || p.isil_code || ''),
      wikidata: String(p.wikidata || p.wikidata_id || ''),
      rating: Number(p.rating || p.google_rating || 0),
      reviews: Number(p.reviews || p.review_count || 0),
      photoCount: Number(p.photoCount || p.photo_count || 0),
    };
  });
  
  // Extract coordinates for map
  if (type === 'map') {
    data.coordinates = data.institutions
      .filter(i => i.latitude && i.longitude && !isNaN(i.latitude) && !isNaN(i.longitude))
      .map(i => ({
        lat: i.latitude!,
        lng: i.longitude!,
        label: i.name,
        type: i.type,
        data: i,
      }));
  }
  
  return data;
}

// ============================================================================
// Hook Implementation
// ============================================================================

export function useMultiDatabaseRAG(): UseMultiDatabaseRAGReturn {
  const [isLoading, setIsLoading] = useState(false);
  const [error, setError] = useState<Error | null>(null);
  const [lastContext, setLastContext] = useState<RAGContext | null>(null);
  const [cacheEnabled, setCacheEnabled] = useState(true);
  const [lastCacheLookup, setLastCacheLookup] = useState<CacheLookupResult | null>(null);
  
  // Initialize cache on mount
  const cacheInitialized = useRef(false);
  useEffect(() => {
    if (!cacheInitialized.current) {
      semanticCache.initialize().then(() => {
        cacheInitialized.current = true;
        console.log('[useMultiDatabaseRAG] Semantic cache initialized');
      }).catch(err => {
        console.error('[useMultiDatabaseRAG] Cache init failed:', err);
      });
    }
  }, []);
  
  /**
   * Main RAG query function - orchestrates multi-database retrieval
   * Now with semantic caching for API cost savings
   */
  const queryRAG = useCallback(async (
    question: string,
    options: RAGOptions = {}
  ): Promise<RAGResponse> => {
    setIsLoading(true);
    setError(null);
    setLastCacheLookup(null);
    
    const {
      maxQdrantResults = 20,
      maxSparqlResults = 50,
      maxTypeDBResults = 50,
      includeSparql = true,
      includeTypeDB = false,  // Disabled by default (may not be running)
      useCache = true,
      bypassCache = false,
      storeInCache = true,
    } = options;
    
    try {
      // Step 1: Generate embedding for the query (needed for both cache lookup and Qdrant search)
      const queryEmbedding = await generateEmbedding(question);
      
      // Step 2: Check semantic cache (if enabled and not bypassed)
      if (cacheEnabled && useCache && !bypassCache) {
        try {
          const cacheResult = await semanticCache.lookup(
            question,
            queryEmbedding,
            { language: options.language, model: options.model }
          );
          
          setLastCacheLookup(cacheResult);
          
          if (cacheResult.found && cacheResult.entry) {
            console.log(
              `[useMultiDatabaseRAG] Cache HIT! Similarity: ${cacheResult.similarity.toFixed(3)}, ` +
              `Method: ${cacheResult.method}, Saved API call!`
            );
            
            // Reconstruct RAGResponse from cached data
            const cached = cacheResult.entry.response;
            
            // Update context state
            setLastContext(cached.context as RAGContext);
            
            return {
              answer: cached.answer,
              sparqlQuery: cached.sparqlQuery,
              typeqlQuery: cached.typeqlQuery,
              context: cached.context as RAGContext,
              visualizationType: cached.visualizationType as VisualizationType || 'card',
              visualizationData: cached.visualizationData as VisualizationData,
              sources: cached.sources as RAGSource[],
              confidence: cached.confidence,
              // Include new fields for person/institution visualization
              retrievedResults: cached.retrievedResults as RetrievedResult[] | undefined,
              queryType: cached.queryType as QueryType | undefined,
              // Add cache metadata to indicate this was from cache
              _fromCache: true,
              _cacheMethod: cacheResult.method,
              _cacheSimilarity: cacheResult.similarity,
            } as RAGResponse & { _fromCache?: boolean; _cacheMethod?: string; _cacheSimilarity?: number };
          }
        } catch (cacheError) {
          console.warn('[useMultiDatabaseRAG] Cache lookup failed, continuing without cache:', cacheError);
        }
      }
      
      // Step 3: Cache miss - perform actual database queries
      // Parallel retrieval from all databases
      const retrievalPromises: Promise<unknown>[] = [
        qdrantSearch(question, maxQdrantResults),
      ];
      
      // Add SPARQL if enabled (construct a basic query from keywords)
      if (includeSparql) {
        const keywords = question.split(/\s+/).filter(w => w.length > 2).slice(0, 3);
        const sparqlSearchQuery = `
          PREFIX rdfs: <http://www.w3.org/2000/01/rdf-schema#>
          PREFIX schema: <http://schema.org/>
          PREFIX skos: <http://www.w3.org/2004/02/skos/core#>
          
          SELECT ?s ?label ?type WHERE {
            ?s rdfs:label|schema:name|skos:prefLabel ?label .
            OPTIONAL { ?s a ?type }
            FILTER(CONTAINS(LCASE(STR(?label)), "${keywords[0]?.toLowerCase() || ''}"))
          }
          LIMIT ${maxSparqlResults}
        `;
        retrievalPromises.push(
          sparqlQuery(sparqlSearchQuery).catch(() => [])
        );
      }
      
      // Add TypeDB if enabled
      if (includeTypeDB) {
        const typeqlSearchQuery = `match $x isa heritage_custodian, has name $n; get $x, $n; limit ${maxTypeDBResults};`;
        retrievalPromises.push(
          typedbQuery(typeqlSearchQuery).catch(() => [])
        );
      }
      
      // Wait for all retrievals
      const results = await Promise.all(retrievalPromises);
      
      const qdrantResults = results[0] as QdrantSearchResult[];
      const sparqlResults = (includeSparql ? results[1] : []) as Record<string, unknown>[];
      const typedbResults = (includeTypeDB ? results[2] || results[1] : []) as Record<string, unknown>[];
      
      const context: RAGContext = {
        qdrantResults,
        sparqlResults,
        typedbResults,
        totalRetrieved: qdrantResults.length + sparqlResults.length + typedbResults.length,
      };
      
      setLastContext(context);
      
      // Call DSPy to generate response
      const dspyResponse = await callDSPy(question, context, options);
      
      // Detect visualization type
      const vizType = dspyResponse.visualizationType || detectVisualizationType(question, context);
      
      // Extract visualization data
      const vizData = extractVisualizationData(vizType, context);
      
      // Build sources list
      const sources: RAGSource[] = [
        ...qdrantResults.slice(0, 5).map(r => ({
          database: 'qdrant' as const,
          id: String(r.id),
          name: String(r.payload?.name || r.payload?.custodian_name || ''),
          score: r.score,
          snippet: String(r.payload?.description || '').slice(0, 200),
        })),
      ];
      
      const response: RAGResponse = {
        answer: dspyResponse.answer,
        sparqlQuery: dspyResponse.sparqlQuery,
        typeqlQuery: dspyResponse.typeqlQuery,
        context,
        visualizationType: vizType,
        visualizationData: vizData,
        sources,
        confidence: dspyResponse.confidence,
        // New fields from backend for person/institution visualization
        retrievedResults: dspyResponse.retrievedResults,
        queryType: dspyResponse.queryType,
      };
      
      // Step 4: Store in cache (if enabled and response is valid)
      // Don't cache error responses (confidence: 0) - these are transient API errors
      if (cacheEnabled && storeInCache && response.confidence > 0) {
        try {
          const cacheResponse: CachedResponse = {
            answer: response.answer,
            sparqlQuery: response.sparqlQuery,
            typeqlQuery: response.typeqlQuery,
            visualizationType: response.visualizationType,
            visualizationData: response.visualizationData,
            sources: response.sources,
            confidence: response.confidence,
            context: response.context,
            // Include new fields for person/institution visualization
            retrievedResults: response.retrievedResults,
            queryType: response.queryType,
          };
          
          await semanticCache.store(
            question,
            queryEmbedding,
            cacheResponse,
            { language: options.language, model: options.model }
          );
          
          console.log('[useMultiDatabaseRAG] Response cached for future queries');
        } catch (cacheError) {
          console.warn('[useMultiDatabaseRAG] Failed to cache response:', cacheError);
        }
      }
      
      return response;
      
    } catch (err) {
      const error = err instanceof Error ? err : new Error('RAG query failed');
      setError(error);
      throw error;
    } finally {
      setIsLoading(false);
    }
  }, [cacheEnabled]);
  
  /**
   * Direct Qdrant search (for debugging/advanced use)
   */
  const searchQdrant = useCallback(async (
    query: string,
    limit: number = 10
  ): Promise<QdrantSearchResult[]> => {
    return qdrantSearch(query, limit);
  }, []);
  
  /**
   * Direct SPARQL query (for debugging/advanced use)
   */
  const querySparql = useCallback(async (
    sparql: string
  ): Promise<Record<string, unknown>[]> => {
    return sparqlQuery(sparql);
  }, []);
  
  /**
   * Direct TypeDB query (for debugging/advanced use)
   */
  const queryTypeDB = useCallback(async (
    typeql: string
  ): Promise<Record<string, unknown>[]> => {
    return typedbQuery(typeql);
  }, []);
  
  /**
   * Clear cached context
   */
  const clearContext = useCallback(() => {
    setLastContext(null);
    setError(null);
    setLastCacheLookup(null);
  }, []);
  
  /**
   * Get cache statistics
   */
  const getCacheStats = useCallback(async (): Promise<CacheStats> => {
    return semanticCache.getStats();
  }, []);
  
  /**
   * Clear the semantic cache
   * @returns Object indicating which caches were cleared
   */
  const clearCache = useCallback(async (): Promise<{ localCleared: boolean; sharedCleared: boolean }> => {
    const result = await semanticCache.clear();
    console.log('[useMultiDatabaseRAG] Semantic cache cleared:', result);
    return result;
  }, []);
  
  /**
   * Update cache similarity threshold
   */
  const setCacheSimilarityThreshold = useCallback((threshold: number): void => {
    semanticCache.setConfig({ similarityThreshold: threshold });
    console.log(`[useMultiDatabaseRAG] Cache similarity threshold set to ${threshold}`);
  }, []);
  
  return {
    isLoading,
    error,
    lastContext,
    cacheEnabled,
    lastCacheLookup,
    queryRAG,
    searchQdrant,
    querySparql,
    queryTypeDB,
    clearContext,
    detectVisualizationType,
    setCacheEnabled,
    getCacheStats,
    clearCache,
    setCacheSimilarityThreshold,
  };
}

export default useMultiDatabaseRAG;