import { test, expect } from '@playwright/test' import { loginAndNavigate, waitForChatReady, getChatInput, askQuestion } from './auth.setup' /** * Cache behavior tests for ArchiefAssistent * * Tests verify that: * - Repeat queries show cache hit indicator * - Cached responses are faster than initial queries * - Cache works correctly across the session */ /** * Helper to submit a query and measure response time * Reuses the working askQuestion helper and adds timing */ async function askQuestionTimed(page: any, question: string): Promise<{ response: string; timeMs: number }> { const startTime = Date.now() const response = await askQuestion(page, question) const endTime = Date.now() return { response, timeMs: endTime - startTime } } test.describe('Cache Behavior', () => { test.beforeEach(async ({ page }) => { await loginAndNavigate(page) await waitForChatReady(page) }) test('should show cache indicator on repeat query', async ({ page }) => { const query = 'Hoeveel archieven zijn er in Utrecht?' // First query await askQuestionTimed(page, query) // Wait a moment await page.waitForTimeout(1000) // Same query again await askQuestionTimed(page, query) // Look for cache indicator in the UI // This depends on implementation - common patterns: // - Text like "cached", "uit cache", "cache hit" // - An icon or badge // - A tooltip const cacheIndicators = [ page.getByText(/cache/i), page.getByText(/cached/i), page.getByText(/uit cache/i), page.locator('[class*="cache"]'), ] let cacheIndicatorFound = false for (const indicator of cacheIndicators) { if (await indicator.isVisible().catch(() => false)) { cacheIndicatorFound = true break } } // Log whether cache indicator was found (not a hard failure if not found) if (!cacheIndicatorFound) { console.log('Note: No visible cache indicator found. May be implemented differently or not at all.') } }) test('cached response should be faster than initial query', async ({ page }) => { const query = 'Hoeveel musea zijn er in Noord-Holland?' // First query (should hit the RAG pipeline) const first = await askQuestionTimed(page, query) console.log(`First query time: ${first.timeMs}ms`) // Wait a moment await page.waitForTimeout(500) // Same query again (should be cached) const second = await askQuestionTimed(page, query) console.log(`Second query time: ${second.timeMs}ms`) // Cached response should be significantly faster // We use a generous threshold since network variability exists // Cache hit should be at least 50% faster or under 2 seconds const isFaster = second.timeMs < first.timeMs * 0.75 || second.timeMs < 2000 // Soft assertion - log the result but don't fail if not met // (caching behavior may vary based on server state) if (isFaster) { console.log('Cache appears to be working - second query was faster') } else { console.log('Warning: Second query was not significantly faster. Cache may not be active.') } // Both queries should return the same response expect(second.response).toBe(first.response) }) test('different queries should not share cache', async ({ page }) => { const query1 = 'Hoeveel archieven zijn er in Gelderland?' const query2 = 'Hoeveel musea zijn er in Gelderland?' // First query const result1 = await askQuestionTimed(page, query1) // Different query const result2 = await askQuestionTimed(page, query2) // Responses should be different (different institution types) expect(result2.response).not.toBe(result1.response) }) test('slight variations should not hit cache', async ({ page }) => { // Two queries that mean the same thing but are phrased differently const query1 = 'Hoeveel archieven zijn er in Utrecht?' const query2 = 'Wat is het aantal archieven in Utrecht?' // First query const result1 = await askQuestionTimed(page, query1) // Wait await page.waitForTimeout(500) // Similar query with different phrasing const result2 = await askQuestionTimed(page, query2) // Both should return similar counts (within reason) // Extract numbers from responses const num1Match = result1.response.match(/\d+/) const num2Match = result2.response.match(/\d+/) if (num1Match && num2Match) { const num1 = parseInt(num1Match[0]) const num2 = parseInt(num2Match[0]) // Should be the same count expect(num2).toBe(num1) } }) test('subtype queries should not share cache with generic type queries', async ({ page }) => { // Rule 46: Ontology-driven cache segmentation // "kunstmusea" (subtype: ART_MUSEUM) should NOT match "musea" (generic type: M) cache const queryGeneric = 'Hoeveel musea zijn er in Amsterdam?' const querySubtype = 'Hoeveel kunstmusea zijn er in Amsterdam?' // First query - generic museum const result1 = await askQuestionTimed(page, queryGeneric) console.log(`Generic "musea" query: ${result1.response}`) // Wait await page.waitForTimeout(500) // Second query - specific subtype (art museum) const result2 = await askQuestionTimed(page, querySubtype) console.log(`Subtype "kunstmusea" query: ${result2.response}`) // Extract numbers from responses const num1Match = result1.response.match(/\d+/) const num2Match = result2.response.match(/\d+/) if (num1Match && num2Match) { const num1 = parseInt(num1Match[0]) const num2 = parseInt(num2Match[0]) // Subtype count should be LESS than or equal to generic count // (kunstmusea is a subset of musea) // If cache is incorrectly shared, they would be the same count console.log(`Generic musea count: ${num1}, Kunstmusea count: ${num2}`) // Art museums should be fewer than total museums (or equal if all are art museums) expect(num2).toBeLessThanOrEqual(num1) // If both are non-zero and equal, that's suspicious - log warning if (num1 > 0 && num2 > 0 && num1 === num2) { console.log('Warning: Counts are identical. Verify cache segmentation is working correctly.') } } }) }) test.describe('Cache with Session', () => { test('cache should persist within same session', async ({ page }) => { await loginAndNavigate(page) await waitForChatReady(page) const query = 'Hoeveel bibliotheken zijn er in Zuid-Holland?' // First query const chatInput = getChatInput(page) await chatInput.fill(query) await chatInput.press('Enter') // Wait for response const responseLocator = page.locator('p, [class*="message"], [class*="response"]') .filter({ hasText: /instellingen|archieven|musea|bibliotheken|gevonden|\d+/ }) .last() await responseLocator.waitFor({ timeout: 45000 }) const firstText = await responseLocator.textContent() // Navigate away (if there are other pages) and back // For now, just reload and re-query // Note: Full cache persistence across page loads depends on implementation // Ask a different question await chatInput.fill('Wat is een archief?') await chatInput.press('Enter') await page.waitForTimeout(5000) // Wait for response // Ask the original question again await chatInput.fill(query) await chatInput.press('Enter') const repeatLocator = page.locator('p, [class*="message"], [class*="response"]') .filter({ hasText: /instellingen|archieven|musea|bibliotheken|gevonden|\d+/ }) .last() await repeatLocator.waitFor({ timeout: 45000 }) const repeatText = await repeatLocator.textContent() // Should get the same answer expect(repeatText).toBe(firstText) }) })