feat: Add SyncPanel component for database synchronization

- Add SyncPanel component with bilingual (NL/EN) support
- Add relative URL handling for production (bronhouder.nl)
- Integrate SyncPanel into Database page
- Show sync status for all 4 databases (DuckLake, PostgreSQL, Oxigraph, Qdrant)
- Support dry-run mode and file limit options
This commit is contained in:
kempersc 2025-12-12 23:42:22 +01:00
parent 505c12601a
commit 41aace785f
5 changed files with 999 additions and 0 deletions

View file

@ -0,0 +1,388 @@
/**
* SyncPanel Component Styles
*/
.sync-panel {
background: var(--card-bg, #fff);
border-radius: 12px;
padding: 24px;
box-shadow: 0 2px 8px rgba(0, 0, 0, 0.1);
margin-bottom: 24px;
}
.sync-header h2 {
margin: 0 0 8px 0;
font-size: 1.5rem;
color: var(--text-primary, #1a1a2e);
}
.sync-description {
color: var(--text-secondary, #666);
margin: 0 0 20px 0;
}
/* Status Section */
.sync-status-section {
background: var(--bg-subtle, #f8f9fa);
border-radius: 8px;
padding: 16px;
margin-bottom: 20px;
}
.sync-status-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 12px;
}
.sync-status-header h3 {
margin: 0;
font-size: 1rem;
color: var(--text-primary, #1a1a2e);
}
.refresh-button {
background: transparent;
border: 1px solid var(--border-color, #ddd);
border-radius: 6px;
padding: 6px 12px;
font-size: 0.85rem;
cursor: pointer;
transition: all 0.2s;
}
.refresh-button:hover:not(:disabled) {
background: var(--bg-hover, #e9ecef);
}
.refresh-button:disabled {
opacity: 0.5;
cursor: not-allowed;
}
/* Error Message */
.sync-error {
background: #fee2e2;
border: 1px solid #fca5a5;
border-radius: 8px;
padding: 12px 16px;
color: #b91c1c;
font-size: 0.9rem;
margin-bottom: 16px;
display: flex;
align-items: flex-start;
gap: 10px;
}
.error-icon {
font-size: 1.2rem;
}
/* Status Grid */
.sync-status-grid {
display: grid;
grid-template-columns: auto 1fr;
gap: 20px;
align-items: center;
}
.sync-stat {
text-align: center;
padding: 12px 20px;
background: var(--primary-color, #4a90d9);
border-radius: 8px;
color: white;
}
.sync-stat .stat-value {
display: block;
font-size: 2rem;
font-weight: bold;
}
.sync-stat .stat-label {
font-size: 0.85rem;
opacity: 0.9;
}
.sync-databases-status {
display: flex;
flex-wrap: wrap;
gap: 12px;
}
.db-connection-status {
display: flex;
align-items: center;
gap: 6px;
padding: 6px 12px;
background: white;
border-radius: 6px;
border: 1px solid var(--border-color, #ddd);
font-size: 0.9rem;
}
.status-indicator {
font-size: 0.6rem;
}
.status-indicator.connected {
color: #22c55e;
}
.status-indicator.disconnected {
color: #ef4444;
}
.db-name {
font-weight: 500;
text-transform: capitalize;
}
/* Sync Controls */
.sync-controls {
display: flex;
flex-direction: column;
gap: 16px;
margin-bottom: 20px;
}
.sync-options {
display: flex;
flex-wrap: wrap;
gap: 20px;
align-items: center;
}
.checkbox-label {
display: flex;
align-items: center;
gap: 8px;
cursor: pointer;
font-size: 0.95rem;
}
.checkbox-label input[type="checkbox"] {
width: 18px;
height: 18px;
cursor: pointer;
}
.limit-input {
display: flex;
align-items: center;
gap: 8px;
}
.limit-input label {
font-size: 0.95rem;
color: var(--text-secondary, #666);
}
.limit-input input {
width: 100px;
padding: 8px 12px;
border: 1px solid var(--border-color, #ddd);
border-radius: 6px;
font-size: 0.9rem;
}
.limit-input input:focus {
outline: none;
border-color: var(--primary-color, #4a90d9);
box-shadow: 0 0 0 2px rgba(74, 144, 217, 0.2);
}
/* Sync Button */
.sync-button {
background: linear-gradient(135deg, #4a90d9, #357abd);
color: white;
border: none;
border-radius: 8px;
padding: 14px 28px;
font-size: 1rem;
font-weight: 600;
cursor: pointer;
transition: all 0.2s;
display: flex;
align-items: center;
justify-content: center;
gap: 8px;
}
.sync-button:hover:not(:disabled) {
background: linear-gradient(135deg, #357abd, #2b6299);
transform: translateY(-1px);
box-shadow: 0 4px 12px rgba(74, 144, 217, 0.3);
}
.sync-button:disabled {
opacity: 0.6;
cursor: not-allowed;
transform: none;
}
.spinner {
animation: spin 1s linear infinite;
}
@keyframes spin {
from { transform: rotate(0deg); }
to { transform: rotate(360deg); }
}
/* Sync Result */
.sync-result {
border-radius: 12px;
padding: 20px;
background: var(--bg-subtle, #f8f9fa);
border: 2px solid transparent;
}
.sync-result.status-success {
border-color: #22c55e;
background: #f0fdf4;
}
.sync-result.status-failed {
border-color: #ef4444;
background: #fef2f2;
}
.sync-result.status-partial {
border-color: #f59e0b;
background: #fffbeb;
}
.sync-result h3 {
margin: 0 0 16px 0;
font-size: 1.2rem;
}
/* Result Summary */
.result-summary {
display: flex;
gap: 24px;
margin-bottom: 20px;
padding-bottom: 16px;
border-bottom: 1px solid var(--border-color, #ddd);
}
.result-stat {
display: flex;
flex-direction: column;
gap: 4px;
}
.result-stat .stat-label {
font-size: 0.85rem;
color: var(--text-secondary, #666);
}
.result-stat .stat-value {
font-size: 1.25rem;
font-weight: 600;
}
/* Per-database Results */
.result-details {
display: grid;
gap: 12px;
}
.db-result {
background: white;
border-radius: 8px;
padding: 12px 16px;
border: 1px solid var(--border-color, #ddd);
}
.db-result.status-success {
border-left: 4px solid #22c55e;
}
.db-result.status-failed {
border-left: 4px solid #ef4444;
}
.db-result-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 8px;
}
.db-result-header .db-name {
font-weight: 600;
font-size: 1rem;
text-transform: capitalize;
}
.db-result-stats {
display: flex;
gap: 16px;
font-size: 0.9rem;
color: var(--text-secondary, #666);
}
.db-result-error {
margin-top: 8px;
padding: 8px 12px;
background: #fee2e2;
border-radius: 4px;
color: #b91c1c;
font-size: 0.85rem;
}
/* Connection Errors */
.connection-errors {
margin-top: 16px;
padding-top: 16px;
border-top: 1px solid var(--border-color, #ddd);
}
.connection-errors h4 {
margin: 0 0 12px 0;
color: #b91c1c;
font-size: 0.95rem;
}
.connection-error {
padding: 8px 12px;
background: #fee2e2;
border-radius: 6px;
margin-bottom: 8px;
font-size: 0.9rem;
color: #b91c1c;
}
.connection-error .db-name {
font-weight: 600;
}
/* Responsive */
@media (max-width: 768px) {
.sync-status-grid {
grid-template-columns: 1fr;
gap: 16px;
}
.sync-stat {
padding: 16px;
}
.sync-databases-status {
justify-content: center;
}
.result-summary {
flex-direction: column;
gap: 12px;
}
.sync-options {
flex-direction: column;
align-items: flex-start;
}
}

View file

@ -0,0 +1,326 @@
/**
* SyncPanel Component
*
* Provides a UI for triggering and monitoring database sync operations.
* Syncs all databases from data/custodian/*.yaml (single source of truth).
*/
import { useState, useCallback } from 'react';
import './SyncPanel.css';
// Types for sync API responses
interface SyncResultResponse {
database: string;
status: string;
records_processed: number;
records_succeeded: number;
records_failed: number;
duration_seconds: number;
error_message: string | null;
details: Record<string, unknown>;
}
interface SyncAllResponse {
overall_status: string;
dry_run: boolean;
started_at: string;
completed_at: string | null;
duration_seconds: number;
total_databases: number;
databases_succeeded: number;
databases_failed: number;
total_records_processed: number;
total_records_succeeded: number;
total_records_failed: number;
results: Record<string, SyncResultResponse>;
connection_errors: Record<string, string>;
}
interface DatabaseStatus {
database: string;
connected: boolean;
status: Record<string, unknown>;
}
interface AllStatusResponse {
databases: DatabaseStatus[];
yaml_file_count: number;
data_directory: string;
}
// Sync API URL - use relative URL for production, localhost for development
const SYNC_API_URL = import.meta.env.VITE_SYNC_API_URL ||
(window.location.hostname === 'localhost' ? 'http://localhost:8766' : '');
// Bilingual text
const TEXT = {
title: { nl: 'Database Synchronisatie', en: 'Database Synchronization' },
description: {
nl: 'Synchroniseer alle databases vanaf de YAML bronbestanden (Single Source of Truth)',
en: 'Sync all databases from YAML source files (Single Source of Truth)',
},
sourceFiles: { nl: 'Bronbestanden', en: 'Source Files' },
yamlFiles: { nl: 'YAML bestanden', en: 'YAML files' },
syncAll: { nl: 'Synchroniseer Alles', en: 'Sync All' },
dryRun: { nl: 'Test Run (geen wijzigingen)', en: 'Dry Run (no changes)' },
syncing: { nl: 'Bezig met synchroniseren...', en: 'Syncing...' },
success: { nl: 'Succesvol', en: 'Success' },
failed: { nl: 'Mislukt', en: 'Failed' },
partial: { nl: 'Gedeeltelijk', en: 'Partial' },
duration: { nl: 'Duur', en: 'Duration' },
records: { nl: 'Records', en: 'Records' },
processed: { nl: 'Verwerkt', en: 'Processed' },
succeeded: { nl: 'Geslaagd', en: 'Succeeded' },
databases: { nl: 'Databases', en: 'Databases' },
limit: { nl: 'Limiet (optioneel)', en: 'Limit (optional)' },
limitPlaceholder: { nl: 'Alle bestanden', en: 'All files' },
connectionError: { nl: 'Verbindingsfout', en: 'Connection Error' },
apiUnavailable: {
nl: 'Sync API niet beschikbaar. Start de server met: uvicorn backend.sync.main:app --port 8766',
en: 'Sync API unavailable. Start server with: uvicorn backend.sync.main:app --port 8766'
},
refreshStatus: { nl: 'Vernieuw Status', en: 'Refresh Status' },
};
interface SyncPanelProps {
language: 'nl' | 'en';
}
export function SyncPanel({ language }: SyncPanelProps) {
const t = (key: keyof typeof TEXT) => TEXT[key][language];
// State
const [status, setStatus] = useState<AllStatusResponse | null>(null);
const [syncResult, setSyncResult] = useState<SyncAllResponse | null>(null);
const [isSyncing, setIsSyncing] = useState(false);
const [isLoadingStatus, setIsLoadingStatus] = useState(false);
const [error, setError] = useState<string | null>(null);
const [dryRun, setDryRun] = useState(true);
const [limit, setLimit] = useState<string>('');
// Fetch status from API
const fetchStatus = useCallback(async () => {
setIsLoadingStatus(true);
setError(null);
try {
const response = await fetch(`${SYNC_API_URL}/api/sync/status`);
if (!response.ok) {
throw new Error(`HTTP ${response.status}`);
}
const data: AllStatusResponse = await response.json();
setStatus(data);
} catch (err) {
setError(t('apiUnavailable'));
setStatus(null);
} finally {
setIsLoadingStatus(false);
}
}, []);
// Trigger sync
const triggerSync = useCallback(async () => {
setIsSyncing(true);
setError(null);
setSyncResult(null);
try {
const body: { dry_run: boolean; limit?: number } = { dry_run: dryRun };
if (limit && !isNaN(parseInt(limit))) {
body.limit = parseInt(limit);
}
const response = await fetch(`${SYNC_API_URL}/api/sync/all`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
});
if (!response.ok) {
throw new Error(`HTTP ${response.status}: ${await response.text()}`);
}
const data: SyncAllResponse = await response.json();
setSyncResult(data);
// Refresh status after sync
await fetchStatus();
} catch (err) {
setError(err instanceof Error ? err.message : 'Unknown error');
} finally {
setIsSyncing(false);
}
}, [dryRun, limit, fetchStatus]);
// Format duration
const formatDuration = (seconds: number) => {
if (seconds < 60) return `${seconds.toFixed(1)}s`;
const mins = Math.floor(seconds / 60);
const secs = seconds % 60;
return `${mins}m ${secs.toFixed(0)}s`;
};
// Get status class
const getStatusClass = (status: string) => {
switch (status) {
case 'success': return 'status-success';
case 'failed': return 'status-failed';
case 'partial': return 'status-partial';
default: return '';
}
};
return (
<div className="sync-panel">
<div className="sync-header">
<h2>🔄 {t('title')}</h2>
<p className="sync-description">{t('description')}</p>
</div>
{/* Status Section */}
<div className="sync-status-section">
<div className="sync-status-header">
<h3>{t('sourceFiles')}</h3>
<button
className="refresh-button"
onClick={fetchStatus}
disabled={isLoadingStatus}
>
{isLoadingStatus ? '⏳' : '🔄'} {t('refreshStatus')}
</button>
</div>
{error && (
<div className="sync-error">
<span className="error-icon"></span>
{error}
</div>
)}
{status && (
<div className="sync-status-grid">
<div className="sync-stat">
<span className="stat-value">{status.yaml_file_count.toLocaleString()}</span>
<span className="stat-label">{t('yamlFiles')}</span>
</div>
<div className="sync-databases-status">
{status.databases.map((db) => (
<div key={db.database} className={`db-connection-status ${db.connected ? 'connected' : 'disconnected'}`}>
<span className={`status-indicator ${db.connected ? 'connected' : 'disconnected'}`}></span>
<span className="db-name">{db.database}</span>
</div>
))}
</div>
</div>
)}
</div>
{/* Sync Controls */}
<div className="sync-controls">
<div className="sync-options">
<label className="checkbox-label">
<input
type="checkbox"
checked={dryRun}
onChange={(e) => setDryRun(e.target.checked)}
disabled={isSyncing}
/>
{t('dryRun')}
</label>
<div className="limit-input">
<label>{t('limit')}:</label>
<input
type="number"
value={limit}
onChange={(e) => setLimit(e.target.value)}
placeholder={t('limitPlaceholder')}
min="1"
disabled={isSyncing}
/>
</div>
</div>
<button
className="sync-button"
onClick={triggerSync}
disabled={isSyncing || !status}
>
{isSyncing ? (
<>
<span className="spinner"></span>
{t('syncing')}
</>
) : (
<>
🔄 {t('syncAll')}
</>
)}
</button>
</div>
{/* Sync Result */}
{syncResult && (
<div className={`sync-result ${getStatusClass(syncResult.overall_status)}`}>
<h3>
{syncResult.dry_run ? '🧪 Dry Run ' : ''}
{syncResult.overall_status === 'success' && `${t('success')}`}
{syncResult.overall_status === 'failed' && `${t('failed')}`}
{syncResult.overall_status === 'partial' && `⚠️ ${t('partial')}`}
</h3>
<div className="result-summary">
<div className="result-stat">
<span className="stat-label">{t('duration')}</span>
<span className="stat-value">{formatDuration(syncResult.duration_seconds)}</span>
</div>
<div className="result-stat">
<span className="stat-label">{t('databases')}</span>
<span className="stat-value">
{syncResult.databases_succeeded}/{syncResult.total_databases}
</span>
</div>
<div className="result-stat">
<span className="stat-label">{t('records')}</span>
<span className="stat-value">
{syncResult.total_records_succeeded.toLocaleString()}/{syncResult.total_records_processed.toLocaleString()}
</span>
</div>
</div>
{/* Per-database results */}
<div className="result-details">
{Object.entries(syncResult.results).map(([dbName, result]) => (
<div key={dbName} className={`db-result ${getStatusClass(result.status)}`}>
<div className="db-result-header">
<span className="db-name">{dbName}</span>
<span className={`db-status ${result.status}`}>
{result.status === 'success' ? '✅' : '❌'}
</span>
</div>
<div className="db-result-stats">
<span>{result.records_succeeded}/{result.records_processed} {t('records')}</span>
<span>{formatDuration(result.duration_seconds)}</span>
</div>
{result.error_message && (
<div className="db-result-error">{result.error_message}</div>
)}
</div>
))}
</div>
{/* Connection errors */}
{Object.keys(syncResult.connection_errors).length > 0 && (
<div className="connection-errors">
<h4>{t('connectionError')}</h4>
{Object.entries(syncResult.connection_errors).map(([db, error]) => (
<div key={db} className="connection-error">
<span className="db-name">{db}</span>: {error}
</div>
))}
</div>
)}
</div>
)}
</div>
);
}

View file

@ -12,3 +12,4 @@ export { OxigraphPanel } from './OxigraphPanel';
export { QdrantPanel } from './QdrantPanel';
export { EmbeddingProjector } from './EmbeddingProjector';
export type { EmbeddingPoint, ProjectedPoint, ProjectionMethod, ViewMode } from './EmbeddingProjector';
export { SyncPanel } from './SyncPanel';

View file

@ -4389,3 +4389,283 @@ body.resizing-row * {
[data-theme="dark"] .computing-overlay p {
color: #888;
}
/* =================================
Re-index Button and Modal Styles
================================= */
/* Re-index Button */
.reindex-button {
padding: 0.5rem 1rem;
background: linear-gradient(135deg, #dc3545 0%, #c82333 100%);
color: white;
border: none;
border-radius: 6px;
font-size: 0.85rem;
font-weight: 500;
cursor: pointer;
transition: all 0.2s ease;
}
.reindex-button:hover:not(:disabled) {
background: linear-gradient(135deg, #c82333 0%, #bd2130 100%);
transform: translateY(-1px);
box-shadow: 0 2px 8px rgba(220, 53, 69, 0.3);
}
.reindex-button:disabled {
opacity: 0.6;
cursor: not-allowed;
transform: none;
}
/* Modal Overlay */
.modal-overlay {
position: fixed;
top: 0;
left: 0;
right: 0;
bottom: 0;
background: rgba(0, 0, 0, 0.6);
display: flex;
align-items: center;
justify-content: center;
z-index: 1000;
backdrop-filter: blur(4px);
}
.modal-dialog {
background: white;
border-radius: 12px;
padding: 1.5rem;
max-width: 450px;
width: 90%;
box-shadow: 0 20px 60px rgba(0, 0, 0, 0.3);
animation: modalSlideIn 0.2s ease-out;
}
@keyframes modalSlideIn {
from {
opacity: 0;
transform: scale(0.95) translateY(-10px);
}
to {
opacity: 1;
transform: scale(1) translateY(0);
}
}
.modal-dialog h3 {
margin: 0 0 1rem 0;
font-size: 1.25rem;
color: #333;
}
.modal-dialog p {
margin: 0 0 1.5rem 0;
color: #666;
line-height: 1.5;
font-size: 0.95rem;
}
.modal-actions {
display: flex;
gap: 0.75rem;
justify-content: flex-end;
}
.secondary-button {
padding: 0.6rem 1.25rem;
background: #f0f0f0;
color: #333;
border: 1px solid #ddd;
border-radius: 6px;
font-size: 0.9rem;
font-weight: 500;
cursor: pointer;
transition: all 0.15s ease;
}
.secondary-button:hover {
background: #e5e5e5;
border-color: #ccc;
}
.danger-button {
padding: 0.6rem 1.25rem;
background: linear-gradient(135deg, #dc3545 0%, #c82333 100%);
color: white;
border: none;
border-radius: 6px;
font-size: 0.9rem;
font-weight: 500;
cursor: pointer;
transition: all 0.15s ease;
}
.danger-button:hover {
background: linear-gradient(135deg, #c82333 0%, #bd2130 100%);
}
/* Reindex Progress Banner */
.reindex-progress-banner {
background: linear-gradient(135deg, #fff8e1 0%, #ffecb3 100%);
border: 1px solid #ffc107;
border-radius: 8px;
padding: 1rem 1.25rem;
margin-bottom: 1rem;
}
.progress-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 0.5rem;
}
.progress-title {
font-weight: 600;
color: #856404;
font-size: 0.95rem;
}
.progress-percent {
font-weight: 700;
color: #856404;
font-size: 1rem;
}
.progress-bar-container {
width: 100%;
height: 8px;
background: rgba(255, 193, 7, 0.3);
border-radius: 4px;
overflow: hidden;
margin-bottom: 0.5rem;
}
.progress-bar-fill {
height: 100%;
background: linear-gradient(90deg, #ffc107 0%, #ffca28 100%);
border-radius: 4px;
transition: width 0.3s ease;
}
.progress-details {
display: flex;
justify-content: space-between;
font-size: 0.8rem;
color: #856404;
}
/* Reindex Result Banner */
.reindex-result-banner {
display: flex;
align-items: center;
gap: 0.75rem;
padding: 0.875rem 1rem;
border-radius: 8px;
margin-bottom: 1rem;
animation: slideDown 0.3s ease-out;
}
@keyframes slideDown {
from {
opacity: 0;
transform: translateY(-10px);
}
to {
opacity: 1;
transform: translateY(0);
}
}
.reindex-result-banner.success {
background: linear-gradient(135deg, #d4edda 0%, #c3e6cb 100%);
border: 1px solid #28a745;
color: #155724;
}
.reindex-result-banner.error {
background: linear-gradient(135deg, #f8d7da 0%, #f5c6cb 100%);
border: 1px solid #dc3545;
color: #721c24;
}
.result-icon {
font-size: 1.25rem;
font-weight: bold;
}
.result-message {
flex: 1;
font-size: 0.9rem;
}
.dismiss-btn {
background: none;
border: none;
font-size: 1.25rem;
cursor: pointer;
opacity: 0.6;
padding: 0;
line-height: 1;
color: inherit;
transition: opacity 0.15s;
}
.dismiss-btn:hover {
opacity: 1;
}
/* Dark theme for re-index components */
[data-theme="dark"] .modal-dialog {
background: #1e1e2e;
border: 1px solid #333;
}
[data-theme="dark"] .modal-dialog h3 {
color: #e0e0e0;
}
[data-theme="dark"] .modal-dialog p {
color: #aaa;
}
[data-theme="dark"] .secondary-button {
background: #333;
color: #e0e0e0;
border-color: #444;
}
[data-theme="dark"] .secondary-button:hover {
background: #444;
border-color: #555;
}
[data-theme="dark"] .reindex-progress-banner {
background: linear-gradient(135deg, #2d2a1f 0%, #3d3520 100%);
border-color: #6d5c00;
}
[data-theme="dark"] .progress-title,
[data-theme="dark"] .progress-percent,
[data-theme="dark"] .progress-details {
color: #ffc107;
}
[data-theme="dark"] .progress-bar-container {
background: rgba(255, 193, 7, 0.15);
}
[data-theme="dark"] .reindex-result-banner.success {
background: linear-gradient(135deg, #1a3d1f 0%, #1e4d24 100%);
border-color: #28a745;
color: #90ee90;
}
[data-theme="dark"] .reindex-result-banner.error {
background: linear-gradient(135deg, #3d1a1a 0%, #4d1e1e 100%);
border-color: #dc3545;
color: #ffb3b3;
}

View file

@ -15,6 +15,7 @@ import { PostgreSQLPanel } from '../components/database/PostgreSQLPanel';
import { TypeDBPanel } from '../components/database/TypeDBPanel';
import { OxigraphPanel } from '../components/database/OxigraphPanel';
import { QdrantPanel } from '../components/database/QdrantPanel';
import { SyncPanel } from '../components/database/SyncPanel';
import { useDuckDB } from '../hooks/useDuckDB';
import { useDuckLake } from '../hooks/useDuckLake';
import { usePostgreSQL } from '../hooks/usePostgreSQL';
@ -345,6 +346,9 @@ function AllDatabasesView({ language, onSelectDatabase }: { language: 'nl' | 'en
return (
<div className="all-databases-view">
{/* Sync Panel - Synchronize YAML files to all databases */}
<SyncPanel language={language} />
{/* Database Cards Grid */}
<section className="comparison-section">
<h2>{t('quickComparison')}</h2>