update postgis data
This commit is contained in:
kempersc
parent
be3fbac601
commit
d4906abae4
1611 changed files with 263326 additions and 755 deletions
|
|
@ -6,6 +6,7 @@ Mounted at /api/geo/ via Caddy reverse proxy.
|
|||
|
||||
Endpoints:
|
||||
- GET / - Health check and geo statistics
|
||||
- GET /countries - Get all countries as GeoJSON with institution counts
|
||||
- GET /provinces - Get all provinces as GeoJSON
|
||||
- GET /municipalities - Get municipalities (with filters)
|
||||
- GET /institutions - Get institutions as GeoJSON (with bbox/type filters)
|
||||
|
|
@ -37,12 +38,12 @@ import asyncpg
|
|||
# ============================================================================
|
||||
|
||||
class GeoSettings(BaseModel):
|
||||
"""PostGIS geo database settings"""
|
||||
"""PostGIS geo database settings - connects to glam_geo with PostGIS boundaries"""
|
||||
host: str = os.getenv("GEO_POSTGRES_HOST", "localhost")
|
||||
port: int = int(os.getenv("GEO_POSTGRES_PORT", "5432"))
|
||||
database: str = os.getenv("GEO_POSTGRES_DB", "glam_geo")
|
||||
database: str = os.getenv("GEO_POSTGRES_DB", "glam_geo") # glam_geo has boundary data
|
||||
user: str = os.getenv("GEO_POSTGRES_USER", "glam_api")
|
||||
password: str = os.getenv("GEO_POSTGRES_PASSWORD", "glam_secret_2025")
|
||||
password: str = os.getenv("GEO_POSTGRES_PASSWORD", "")
|
||||
|
||||
# Server settings
|
||||
api_host: str = os.getenv("GEO_API_HOST", "0.0.0.0")
|
||||
|
|
@ -264,6 +265,122 @@ async def get_provinces(
|
|||
}
|
||||
|
||||
|
||||
@app.get("/countries")
|
||||
async def get_countries(
|
||||
simplified: bool = Query(True, description="Return simplified geometries"),
|
||||
with_counts: bool = Query(False, description="Include institution counts per country"),
|
||||
):
|
||||
"""Get all countries as GeoJSON FeatureCollection with optional institution counts"""
|
||||
pool = await get_pool()
|
||||
|
||||
# Use more aggressive simplification for countries (world view)
|
||||
tolerance = 0.01 if simplified else 0
|
||||
|
||||
async with pool.acquire() as conn:
|
||||
if with_counts:
|
||||
# Join with custodians to get counts per country
|
||||
rows = await conn.fetch(f"""
|
||||
SELECT
|
||||
bc.id,
|
||||
bc.iso_a2 as country_code,
|
||||
bc.iso_a3,
|
||||
bc.country_name as name,
|
||||
ST_AsGeoJSON(
|
||||
{'ST_Simplify(bc.geom, ' + str(tolerance) + ')' if simplified else 'bc.geom'}
|
||||
) as geometry,
|
||||
ST_X(bc.centroid) as centroid_lon,
|
||||
ST_Y(bc.centroid) as centroid_lat,
|
||||
bc.area_km2,
|
||||
COALESCE(counts.institution_count, 0) as institution_count
|
||||
FROM boundary_countries bc
|
||||
LEFT JOIN (
|
||||
SELECT country_code, COUNT(*) as institution_count
|
||||
FROM custodians
|
||||
WHERE country_code IS NOT NULL
|
||||
GROUP BY country_code
|
||||
) counts ON bc.iso_a2 = counts.country_code
|
||||
WHERE bc.geom IS NOT NULL
|
||||
ORDER BY bc.country_name
|
||||
""")
|
||||
else:
|
||||
rows = await conn.fetch(f"""
|
||||
SELECT
|
||||
id,
|
||||
iso_a2 as country_code,
|
||||
iso_a3,
|
||||
country_name as name,
|
||||
ST_AsGeoJSON(
|
||||
{'ST_Simplify(geom, ' + str(tolerance) + ')' if simplified else 'geom'}
|
||||
) as geometry,
|
||||
ST_X(centroid) as centroid_lon,
|
||||
ST_Y(centroid) as centroid_lat,
|
||||
area_km2
|
||||
FROM boundary_countries
|
||||
WHERE geom IS NOT NULL
|
||||
ORDER BY country_name
|
||||
""")
|
||||
|
||||
features = []
|
||||
total_institutions = 0
|
||||
countries_with_data = 0
|
||||
|
||||
for row in rows:
|
||||
# Parse geometry from string to dict (ST_AsGeoJSON returns text)
|
||||
geometry = row['geometry']
|
||||
if geometry is None:
|
||||
# Skip countries with no geometry (e.g., Vatican City)
|
||||
continue
|
||||
if isinstance(geometry, str):
|
||||
geometry = json.loads(geometry)
|
||||
|
||||
# Ensure geometry has required structure
|
||||
if not isinstance(geometry, dict) or 'type' not in geometry or 'coordinates' not in geometry:
|
||||
continue
|
||||
|
||||
iso_a2 = row['country_code'].strip() if row['country_code'] else None
|
||||
iso_a3 = row['iso_a3'].strip() if row['iso_a3'] else None
|
||||
institution_count = row['institution_count'] if with_counts else 0
|
||||
|
||||
# Track totals
|
||||
if with_counts:
|
||||
total_institutions += institution_count
|
||||
if institution_count > 0:
|
||||
countries_with_data += 1
|
||||
|
||||
# Build properties with frontend-expected field names
|
||||
properties = {
|
||||
"id": row['id'],
|
||||
"iso_a2": iso_a2, # Frontend expects iso_a2
|
||||
"iso_a3": iso_a3,
|
||||
"name": row['name'],
|
||||
"institution_count": institution_count,
|
||||
"centroid": [
|
||||
float(row['centroid_lon']) if row['centroid_lon'] else None,
|
||||
float(row['centroid_lat']) if row['centroid_lat'] else None,
|
||||
],
|
||||
"area_km2": float(row['area_km2']) if row['area_km2'] else None,
|
||||
}
|
||||
|
||||
features.append({
|
||||
"type": "Feature",
|
||||
"id": iso_a2,
|
||||
"geometry": geometry,
|
||||
"properties": properties
|
||||
})
|
||||
|
||||
return {
|
||||
"type": "FeatureCollection",
|
||||
"features": features,
|
||||
"metadata": {
|
||||
"count": len(features),
|
||||
"total_institutions": total_institutions,
|
||||
"countries_with_data": countries_with_data,
|
||||
"type_filter": None,
|
||||
"simplified": simplified,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@app.get("/municipalities")
|
||||
async def get_municipalities(
|
||||
province: Optional[str] = Query(None, description="Filter by province ISO code (e.g., NH)"),
|
||||
|
|
@ -325,15 +442,16 @@ async def get_municipalities(
|
|||
@app.get("/institutions")
|
||||
async def get_institutions(
|
||||
bbox: Optional[str] = Query(None, description="Bounding box: minLon,minLat,maxLon,maxLat"),
|
||||
province: Optional[str] = Query(None, description="Filter by province ISO code"),
|
||||
province: Optional[str] = Query(None, description="Filter by province ISO code (e.g., NH, ZH)"),
|
||||
country: Optional[str] = Query(None, description="Filter by country code (e.g., NL, DE, JP)"),
|
||||
type: Optional[str] = Query(None, description="Filter by institution type (G,L,A,M,O,R,C,U,B,E,S,F,I,X,P,H,D,N,T)"),
|
||||
limit: int = Query(2000, ge=1, le=5000, description="Maximum results")
|
||||
limit: int = Query(50000, ge=1, le=100000, description="Maximum results")
|
||||
):
|
||||
"""Get institutions as GeoJSON FeatureCollection"""
|
||||
"""Get institutions as GeoJSON FeatureCollection with full metadata from custodians table"""
|
||||
pool = await get_pool()
|
||||
|
||||
# Build WHERE clauses
|
||||
conditions = ["i.geom IS NOT NULL"]
|
||||
# Build WHERE clauses - query custodians table directly
|
||||
conditions = ["lat IS NOT NULL AND lon IS NOT NULL"]
|
||||
params = []
|
||||
param_count = 0
|
||||
|
||||
|
|
@ -342,7 +460,8 @@ async def get_institutions(
|
|||
min_lon, min_lat, max_lon, max_lat = map(float, bbox.split(','))
|
||||
param_count += 4
|
||||
conditions.append(f"""
|
||||
i.geom && ST_MakeEnvelope(${param_count-3}, ${param_count-2}, ${param_count-1}, ${param_count}, 4326)
|
||||
lon >= ${param_count-3} AND lat >= ${param_count-2}
|
||||
AND lon <= ${param_count-1} AND lat <= ${param_count}
|
||||
""")
|
||||
params.extend([min_lon, min_lat, max_lon, max_lat])
|
||||
except ValueError:
|
||||
|
|
@ -350,36 +469,70 @@ async def get_institutions(
|
|||
|
||||
if province:
|
||||
param_count += 1
|
||||
conditions.append(f"p.iso_code = ${param_count}")
|
||||
conditions.append(f"region_code = ${param_count}")
|
||||
params.append(province.upper())
|
||||
|
||||
if type:
|
||||
param_count += 1
|
||||
conditions.append(f"i.institution_type = ${param_count}")
|
||||
conditions.append(f"type = ${param_count}")
|
||||
params.append(type.upper())
|
||||
|
||||
if country:
|
||||
param_count += 1
|
||||
conditions.append(f"country_code = ${param_count}")
|
||||
params.append(country.upper())
|
||||
|
||||
param_count += 1
|
||||
where_clause = " AND ".join(conditions)
|
||||
|
||||
# Query custodians table with all rich metadata fields
|
||||
query = f"""
|
||||
SELECT
|
||||
i.ghcid_current as ghcid,
|
||||
i.name,
|
||||
i.institution_type as type,
|
||||
i.type_name,
|
||||
ST_X(i.geom) as lon,
|
||||
ST_Y(i.geom) as lat,
|
||||
i.city,
|
||||
p.iso_code as province_iso,
|
||||
p.name as province_name,
|
||||
i.rating,
|
||||
i.total_ratings,
|
||||
i.wikidata_id,
|
||||
i.website
|
||||
FROM institutions i
|
||||
LEFT JOIN provinces p ON i.province_id = p.id
|
||||
ghcid,
|
||||
name,
|
||||
emic_name,
|
||||
type,
|
||||
type_name,
|
||||
lon,
|
||||
lat,
|
||||
city,
|
||||
region as province,
|
||||
region_code as province_iso,
|
||||
country_code,
|
||||
formatted_address,
|
||||
street_address,
|
||||
postal_code,
|
||||
rating,
|
||||
total_ratings,
|
||||
wikidata_id,
|
||||
website,
|
||||
phone,
|
||||
email,
|
||||
isil_code,
|
||||
google_place_id,
|
||||
description,
|
||||
opening_hours,
|
||||
reviews,
|
||||
photos,
|
||||
photo_urls,
|
||||
business_status,
|
||||
street_view_url,
|
||||
founding_year,
|
||||
dissolution_year,
|
||||
temporal_extent,
|
||||
museum_register,
|
||||
youtube_channel_url,
|
||||
youtube_subscriber_count,
|
||||
youtube_video_count,
|
||||
youtube_enrichment,
|
||||
social_facebook,
|
||||
social_twitter,
|
||||
social_instagram,
|
||||
wikidata_label_en,
|
||||
wikidata_description_en
|
||||
FROM custodians
|
||||
WHERE {where_clause}
|
||||
ORDER BY i.name
|
||||
ORDER BY name
|
||||
LIMIT ${param_count}
|
||||
"""
|
||||
|
||||
|
|
@ -390,25 +543,80 @@ async def get_institutions(
|
|||
|
||||
features = []
|
||||
for row in rows:
|
||||
# Build properties with all available metadata
|
||||
props = {
|
||||
"ghcid": row['ghcid'],
|
||||
"name": row['name'],
|
||||
"emic_name": row['emic_name'],
|
||||
"type": row['type'],
|
||||
"type_name": row['type_name'],
|
||||
"city": row['city'],
|
||||
"province": row['province'],
|
||||
"province_iso": row['province_iso'],
|
||||
"country_code": row['country_code'],
|
||||
"formatted_address": row['formatted_address'],
|
||||
"rating": float(row['rating']) if row['rating'] else None,
|
||||
"total_ratings": row['total_ratings'],
|
||||
"wikidata_id": row['wikidata_id'],
|
||||
"website": row['website'],
|
||||
"phone": row['phone'],
|
||||
"email": row['email'],
|
||||
"isil_code": row['isil_code'],
|
||||
"google_place_id": row['google_place_id'],
|
||||
"description": row['description'],
|
||||
"business_status": row['business_status'],
|
||||
"street_view_url": row['street_view_url'],
|
||||
"founding_year": row['founding_year'],
|
||||
"dissolution_year": row['dissolution_year'],
|
||||
}
|
||||
|
||||
# Add JSONB fields (handle potential None values)
|
||||
if row['opening_hours']:
|
||||
props["opening_hours"] = row['opening_hours']
|
||||
if row['reviews']:
|
||||
props["reviews"] = row['reviews']
|
||||
if row['photos']:
|
||||
props["photos"] = row['photos']
|
||||
if row['photo_urls']:
|
||||
props["photo_urls"] = row['photo_urls']
|
||||
if row['temporal_extent']:
|
||||
props["temporal_extent"] = row['temporal_extent']
|
||||
if row['museum_register']:
|
||||
props["museum_register"] = row['museum_register']
|
||||
if row['youtube_enrichment']:
|
||||
props["youtube_enrichment"] = row['youtube_enrichment']
|
||||
elif row['youtube_channel_url']:
|
||||
# Build minimal YouTube data if enrichment not present
|
||||
props["youtube"] = {
|
||||
"channel_url": row['youtube_channel_url'],
|
||||
"subscriber_count": row['youtube_subscriber_count'],
|
||||
"video_count": row['youtube_video_count'],
|
||||
}
|
||||
|
||||
# Social media
|
||||
social = {}
|
||||
if row['social_facebook']:
|
||||
social['facebook'] = row['social_facebook']
|
||||
if row['social_twitter']:
|
||||
social['twitter'] = row['social_twitter']
|
||||
if row['social_instagram']:
|
||||
social['instagram'] = row['social_instagram']
|
||||
if social:
|
||||
props["social_media"] = social
|
||||
|
||||
# Wikidata labels
|
||||
if row['wikidata_label_en']:
|
||||
props["wikidata_label"] = row['wikidata_label_en']
|
||||
if row['wikidata_description_en']:
|
||||
props["wikidata_description"] = row['wikidata_description_en']
|
||||
|
||||
features.append({
|
||||
"type": "Feature",
|
||||
"geometry": {
|
||||
"type": "Point",
|
||||
"coordinates": [float(row['lon']), float(row['lat'])]
|
||||
},
|
||||
"properties": {
|
||||
"ghcid": row['ghcid'],
|
||||
"name": row['name'],
|
||||
"type": row['type'],
|
||||
"type_name": row['type_name'],
|
||||
"city": row['city'],
|
||||
"province_iso": row['province_iso'],
|
||||
"province": row['province_name'],
|
||||
"rating": float(row['rating']) if row['rating'] else None,
|
||||
"total_ratings": row['total_ratings'],
|
||||
"wikidata_id": row['wikidata_id'],
|
||||
"website": row['website'],
|
||||
}
|
||||
"properties": props
|
||||
})
|
||||
|
||||
return {
|
||||
|
|
@ -426,58 +634,234 @@ async def get_institutions(
|
|||
}
|
||||
|
||||
|
||||
@app.get("/institution/{ghcid}", response_model=InstitutionDetail)
|
||||
@app.get("/institution/{ghcid}")
|
||||
async def get_institution(ghcid: str):
|
||||
"""Get detailed information for a single institution"""
|
||||
"""Get detailed information for a single institution with full metadata"""
|
||||
pool = await get_pool()
|
||||
|
||||
async with pool.acquire() as conn:
|
||||
row = await conn.fetchrow("""
|
||||
SELECT
|
||||
i.ghcid_current as ghcid,
|
||||
i.name,
|
||||
i.institution_type as type,
|
||||
i.type_name,
|
||||
ST_X(i.geom) as lon,
|
||||
ST_Y(i.geom) as lat,
|
||||
i.address,
|
||||
i.city,
|
||||
p.name as province,
|
||||
i.website,
|
||||
i.phone,
|
||||
i.wikidata_id,
|
||||
i.rating,
|
||||
i.total_ratings,
|
||||
i.description,
|
||||
i.reviews,
|
||||
i.genealogiewerkbalk
|
||||
FROM institutions i
|
||||
LEFT JOIN provinces p ON i.province_id = p.id
|
||||
WHERE i.ghcid_current = $1
|
||||
ghcid,
|
||||
name,
|
||||
emic_name,
|
||||
verified_name,
|
||||
type,
|
||||
type_name,
|
||||
lon,
|
||||
lat,
|
||||
city,
|
||||
region as province,
|
||||
region_code as province_iso,
|
||||
country_code,
|
||||
formatted_address,
|
||||
street_address,
|
||||
postal_code,
|
||||
website,
|
||||
phone,
|
||||
email,
|
||||
wikidata_id,
|
||||
isil_code,
|
||||
google_place_id,
|
||||
rating,
|
||||
total_ratings,
|
||||
description,
|
||||
business_status,
|
||||
street_view_url,
|
||||
google_maps_url,
|
||||
opening_hours,
|
||||
reviews,
|
||||
photos,
|
||||
photo_urls,
|
||||
founding_year,
|
||||
founding_date,
|
||||
dissolution_year,
|
||||
dissolution_date,
|
||||
temporal_extent,
|
||||
museum_register,
|
||||
youtube_channel_id,
|
||||
youtube_channel_url,
|
||||
youtube_subscriber_count,
|
||||
youtube_video_count,
|
||||
youtube_view_count,
|
||||
youtube_enrichment,
|
||||
social_facebook,
|
||||
social_twitter,
|
||||
social_instagram,
|
||||
social_linkedin,
|
||||
social_youtube,
|
||||
logo_url,
|
||||
wikidata_label_nl,
|
||||
wikidata_label_en,
|
||||
wikidata_description_nl,
|
||||
wikidata_description_en,
|
||||
wikidata_types,
|
||||
wikidata_inception,
|
||||
wikidata_enrichment,
|
||||
genealogiewerkbalk,
|
||||
nan_isil_enrichment,
|
||||
kb_enrichment,
|
||||
zcbs_enrichment,
|
||||
web_claims,
|
||||
ghcid_uuid,
|
||||
ghcid_numeric,
|
||||
identifiers,
|
||||
data_source,
|
||||
data_tier,
|
||||
provenance
|
||||
FROM custodians
|
||||
WHERE ghcid = $1
|
||||
""", ghcid)
|
||||
|
||||
if not row:
|
||||
raise HTTPException(status_code=404, detail=f"Institution '{ghcid}' not found")
|
||||
|
||||
return InstitutionDetail(
|
||||
ghcid=row['ghcid'],
|
||||
name=row['name'],
|
||||
type=row['type'],
|
||||
type_name=row['type_name'],
|
||||
lat=float(row['lat']) if row['lat'] else None,
|
||||
lon=float(row['lon']) if row['lon'] else None,
|
||||
address=row['address'],
|
||||
city=row['city'],
|
||||
province=row['province'],
|
||||
website=row['website'],
|
||||
phone=row['phone'],
|
||||
wikidata_id=row['wikidata_id'],
|
||||
rating=float(row['rating']) if row['rating'] else None,
|
||||
total_ratings=row['total_ratings'],
|
||||
description=row['description'],
|
||||
reviews=row['reviews'],
|
||||
genealogiewerkbalk=row['genealogiewerkbalk'],
|
||||
)
|
||||
# Build comprehensive response with all metadata
|
||||
result = {
|
||||
"ghcid": row['ghcid'],
|
||||
"name": row['name'],
|
||||
"emic_name": row['emic_name'],
|
||||
"verified_name": row['verified_name'],
|
||||
"type": row['type'],
|
||||
"type_name": row['type_name'],
|
||||
"lat": float(row['lat']) if row['lat'] else None,
|
||||
"lon": float(row['lon']) if row['lon'] else None,
|
||||
"city": row['city'],
|
||||
"province": row['province'],
|
||||
"province_iso": row['province_iso'],
|
||||
"country_code": row['country_code'],
|
||||
"formatted_address": row['formatted_address'],
|
||||
"street_address": row['street_address'],
|
||||
"postal_code": row['postal_code'],
|
||||
"website": row['website'],
|
||||
"phone": row['phone'],
|
||||
"email": row['email'],
|
||||
"wikidata_id": row['wikidata_id'],
|
||||
"isil_code": row['isil_code'],
|
||||
"google_place_id": row['google_place_id'],
|
||||
"rating": float(row['rating']) if row['rating'] else None,
|
||||
"total_ratings": row['total_ratings'],
|
||||
"description": row['description'],
|
||||
"business_status": row['business_status'],
|
||||
"street_view_url": row['street_view_url'],
|
||||
"google_maps_url": row['google_maps_url'],
|
||||
}
|
||||
|
||||
# JSONB fields - only include if present
|
||||
if row['opening_hours']:
|
||||
result["opening_hours"] = row['opening_hours']
|
||||
if row['reviews']:
|
||||
result["reviews"] = row['reviews']
|
||||
if row['photos']:
|
||||
result["photos"] = row['photos']
|
||||
if row['photo_urls']:
|
||||
result["photo_urls"] = row['photo_urls']
|
||||
if row['identifiers']:
|
||||
result["identifiers"] = row['identifiers']
|
||||
|
||||
# Temporal data
|
||||
temporal = {}
|
||||
if row['founding_year']:
|
||||
temporal["founding_year"] = row['founding_year']
|
||||
if row['founding_date']:
|
||||
temporal["founding_date"] = row['founding_date'].isoformat() if row['founding_date'] else None
|
||||
if row['dissolution_year']:
|
||||
temporal["dissolution_year"] = row['dissolution_year']
|
||||
if row['dissolution_date']:
|
||||
temporal["dissolution_date"] = row['dissolution_date'].isoformat() if row['dissolution_date'] else None
|
||||
if row['temporal_extent']:
|
||||
temporal["extent"] = row['temporal_extent']
|
||||
if temporal:
|
||||
result["temporal"] = temporal
|
||||
|
||||
# Museum register
|
||||
if row['museum_register']:
|
||||
result["museum_register"] = row['museum_register']
|
||||
|
||||
# YouTube enrichment
|
||||
youtube = {}
|
||||
if row['youtube_channel_id']:
|
||||
youtube["channel_id"] = row['youtube_channel_id']
|
||||
if row['youtube_channel_url']:
|
||||
youtube["channel_url"] = row['youtube_channel_url']
|
||||
if row['youtube_subscriber_count']:
|
||||
youtube["subscriber_count"] = row['youtube_subscriber_count']
|
||||
if row['youtube_video_count']:
|
||||
youtube["video_count"] = row['youtube_video_count']
|
||||
if row['youtube_view_count']:
|
||||
youtube["view_count"] = row['youtube_view_count']
|
||||
if row['youtube_enrichment']:
|
||||
youtube["enrichment"] = row['youtube_enrichment']
|
||||
if youtube:
|
||||
result["youtube"] = youtube
|
||||
|
||||
# Social media
|
||||
social = {}
|
||||
if row['social_facebook']:
|
||||
social["facebook"] = row['social_facebook']
|
||||
if row['social_twitter']:
|
||||
social["twitter"] = row['social_twitter']
|
||||
if row['social_instagram']:
|
||||
social["instagram"] = row['social_instagram']
|
||||
if row['social_linkedin']:
|
||||
social["linkedin"] = row['social_linkedin']
|
||||
if row['social_youtube']:
|
||||
social["youtube"] = row['social_youtube']
|
||||
if social:
|
||||
result["social_media"] = social
|
||||
|
||||
# Wikidata
|
||||
wikidata = {}
|
||||
if row['wikidata_label_nl']:
|
||||
wikidata["label_nl"] = row['wikidata_label_nl']
|
||||
if row['wikidata_label_en']:
|
||||
wikidata["label_en"] = row['wikidata_label_en']
|
||||
if row['wikidata_description_nl']:
|
||||
wikidata["description_nl"] = row['wikidata_description_nl']
|
||||
if row['wikidata_description_en']:
|
||||
wikidata["description_en"] = row['wikidata_description_en']
|
||||
if row['wikidata_types']:
|
||||
wikidata["types"] = row['wikidata_types']
|
||||
if row['wikidata_inception']:
|
||||
wikidata["inception"] = row['wikidata_inception']
|
||||
if row['wikidata_enrichment']:
|
||||
wikidata["enrichment"] = row['wikidata_enrichment']
|
||||
if wikidata:
|
||||
result["wikidata"] = wikidata
|
||||
|
||||
# Logo
|
||||
if row['logo_url']:
|
||||
result["logo_url"] = row['logo_url']
|
||||
|
||||
# Other enrichment data
|
||||
if row['genealogiewerkbalk']:
|
||||
result["genealogiewerkbalk"] = row['genealogiewerkbalk']
|
||||
if row['nan_isil_enrichment']:
|
||||
result["nan_isil_enrichment"] = row['nan_isil_enrichment']
|
||||
if row['kb_enrichment']:
|
||||
result["kb_enrichment"] = row['kb_enrichment']
|
||||
if row['zcbs_enrichment']:
|
||||
result["zcbs_enrichment"] = row['zcbs_enrichment']
|
||||
if row['web_claims']:
|
||||
result["web_claims"] = row['web_claims']
|
||||
|
||||
# GHCID details
|
||||
ghcid_data = {"current": row['ghcid']}
|
||||
if row['ghcid_uuid']:
|
||||
ghcid_data["uuid"] = str(row['ghcid_uuid'])
|
||||
if row['ghcid_numeric']:
|
||||
ghcid_data["numeric"] = int(row['ghcid_numeric'])
|
||||
result["ghcid_details"] = ghcid_data
|
||||
|
||||
# Provenance
|
||||
if row['data_source'] or row['data_tier'] or row['provenance']:
|
||||
result["provenance"] = {
|
||||
"data_source": row['data_source'],
|
||||
"data_tier": row['data_tier'],
|
||||
"details": row['provenance'],
|
||||
}
|
||||
|
||||
return result
|
||||
|
||||
|
||||
@app.get("/search")
|
||||
|
|
|
|||
18
backend/valkey/Dockerfile
Normal file
18
backend/valkey/Dockerfile
Normal file
|
|
@ -0,0 +1,18 @@
|
|||
# Dockerfile for GLAM Valkey Cache API
|
||||
FROM python:3.11-slim
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Install curl for healthcheck
|
||||
RUN apt-get update && apt-get install -y curl && rm -rf /var/lib/apt/lists/*
|
||||
|
||||
# Install Python dependencies
|
||||
COPY requirements.txt .
|
||||
RUN pip install --no-cache-dir -r requirements.txt
|
||||
|
||||
# Copy application
|
||||
COPY main.py .
|
||||
|
||||
# Run with uvicorn
|
||||
EXPOSE 8090
|
||||
CMD ["uvicorn", "main:app", "--host", "0.0.0.0", "--port", "8090"]
|
||||
70
backend/valkey/docker-compose.yaml
Normal file
70
backend/valkey/docker-compose.yaml
Normal file
|
|
@ -0,0 +1,70 @@
|
|||
# Docker Compose for GLAM Valkey Semantic Cache
|
||||
#
|
||||
# This deploys:
|
||||
# - Valkey (Redis-compatible) for shared cache storage
|
||||
# - FastAPI backend for cache API
|
||||
#
|
||||
# Usage:
|
||||
# docker-compose up -d
|
||||
# docker-compose logs -f valkey-api
|
||||
|
||||
version: '3.8'
|
||||
|
||||
services:
|
||||
# Valkey - Redis-compatible in-memory store
|
||||
valkey:
|
||||
image: valkey/valkey:8-alpine
|
||||
container_name: glam-valkey
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:6379:6379" # Only bind to localhost
|
||||
volumes:
|
||||
- valkey-data:/data
|
||||
command: >
|
||||
valkey-server
|
||||
--appendonly yes
|
||||
--maxmemory 512mb
|
||||
--maxmemory-policy allkeys-lru
|
||||
--save 60 1
|
||||
healthcheck:
|
||||
test: ["CMD", "valkey-cli", "ping"]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
networks:
|
||||
- glam-cache
|
||||
|
||||
# FastAPI Cache API
|
||||
valkey-api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Dockerfile
|
||||
container_name: glam-valkey-api
|
||||
restart: unless-stopped
|
||||
ports:
|
||||
- "127.0.0.1:8090:8090" # Only bind to localhost, Caddy will proxy
|
||||
environment:
|
||||
- VALKEY_HOST=valkey
|
||||
- VALKEY_PORT=6379
|
||||
- VALKEY_DB=0
|
||||
- CACHE_TTL_SECONDS=86400
|
||||
- MAX_CACHE_ENTRIES=10000
|
||||
- SIMILARITY_THRESHOLD=0.92
|
||||
depends_on:
|
||||
valkey:
|
||||
condition: service_healthy
|
||||
healthcheck:
|
||||
test: ["CMD", "curl", "-f", "http://localhost:8090/health"]
|
||||
interval: 30s
|
||||
timeout: 10s
|
||||
retries: 3
|
||||
networks:
|
||||
- glam-cache
|
||||
|
||||
volumes:
|
||||
valkey-data:
|
||||
name: glam-valkey-data
|
||||
|
||||
networks:
|
||||
glam-cache:
|
||||
name: glam-cache-network
|
||||
607
backend/valkey/main.py
Normal file
607
backend/valkey/main.py
Normal file
|
|
@ -0,0 +1,607 @@
|
|||
"""
|
||||
Valkey/Redis Semantic Cache Backend Service
|
||||
|
||||
Provides a shared cache layer for RAG query responses across all users.
|
||||
Uses vector similarity search for semantic matching.
|
||||
|
||||
Architecture:
|
||||
- Two-tier caching: Client (IndexedDB) -> Server (Valkey)
|
||||
- Embeddings stored as binary vectors for efficient similarity search
|
||||
- TTL-based expiration with LRU eviction
|
||||
- Optional: Use Redis Stack's vector search (RediSearch) for native similarity
|
||||
|
||||
Endpoints:
|
||||
- POST /cache/lookup - Find semantically similar cached queries
|
||||
- POST /cache/store - Store a query/response pair
|
||||
- DELETE /cache/clear - Clear all cache entries
|
||||
- GET /cache/stats - Get cache statistics
|
||||
- GET /health - Health check
|
||||
|
||||
@author TextPast / NDE
|
||||
@version 1.0.0
|
||||
"""
|
||||
|
||||
import os
|
||||
import json
|
||||
import time
|
||||
import hashlib
|
||||
import struct
|
||||
from typing import Optional, List, Dict, Any
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
import numpy as np
|
||||
from fastapi import FastAPI, HTTPException, Query
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from pydantic import BaseModel, Field
|
||||
import redis.asyncio as redis
|
||||
|
||||
# =============================================================================
|
||||
# Configuration
|
||||
# =============================================================================
|
||||
|
||||
VALKEY_HOST = os.getenv("VALKEY_HOST", "localhost")
|
||||
VALKEY_PORT = int(os.getenv("VALKEY_PORT", "6379"))
|
||||
VALKEY_PASSWORD = os.getenv("VALKEY_PASSWORD", None)
|
||||
VALKEY_DB = int(os.getenv("VALKEY_DB", "0"))
|
||||
|
||||
# Cache settings
|
||||
CACHE_PREFIX = "glam:semantic_cache:"
|
||||
CACHE_TTL_SECONDS = int(os.getenv("CACHE_TTL_SECONDS", "86400")) # 24 hours
|
||||
MAX_CACHE_ENTRIES = int(os.getenv("MAX_CACHE_ENTRIES", "10000"))
|
||||
SIMILARITY_THRESHOLD = float(os.getenv("SIMILARITY_THRESHOLD", "0.92"))
|
||||
|
||||
# Embedding dimension (for validation)
|
||||
EMBEDDING_DIM = int(os.getenv("EMBEDDING_DIM", "1536")) # OpenAI ada-002 default
|
||||
|
||||
# =============================================================================
|
||||
# Models
|
||||
# =============================================================================
|
||||
|
||||
class CachedResponse(BaseModel):
|
||||
"""The RAG response to cache"""
|
||||
answer: str
|
||||
sparql_query: Optional[str] = None
|
||||
typeql_query: Optional[str] = None
|
||||
visualization_type: Optional[str] = None
|
||||
visualization_data: Optional[Any] = None
|
||||
sources: List[Any] = Field(default_factory=list)
|
||||
confidence: float = 0.0
|
||||
context: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class CacheLookupRequest(BaseModel):
|
||||
"""Request to look up a query in cache"""
|
||||
query: str
|
||||
embedding: Optional[List[float]] = None
|
||||
language: str = "nl"
|
||||
similarity_threshold: Optional[float] = None
|
||||
|
||||
|
||||
class CacheStoreRequest(BaseModel):
|
||||
"""Request to store a query/response in cache"""
|
||||
query: str
|
||||
embedding: Optional[List[float]] = None
|
||||
response: CachedResponse
|
||||
language: str = "nl"
|
||||
model: str = "unknown"
|
||||
|
||||
|
||||
class CacheLookupResponse(BaseModel):
|
||||
"""Response from cache lookup"""
|
||||
found: bool
|
||||
similarity: float = 0.0
|
||||
method: str = "none" # 'semantic', 'fuzzy', 'exact', 'none'
|
||||
lookup_time_ms: float = 0.0
|
||||
entry: Optional[Dict[str, Any]] = None
|
||||
|
||||
|
||||
class CacheStats(BaseModel):
|
||||
"""Cache statistics"""
|
||||
total_entries: int
|
||||
total_hits: int
|
||||
total_misses: int
|
||||
hit_rate: float
|
||||
storage_used_bytes: int
|
||||
oldest_entry: Optional[int] = None
|
||||
newest_entry: Optional[int] = None
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Utility Functions
|
||||
# =============================================================================
|
||||
|
||||
def normalize_query(query: str) -> str:
|
||||
"""Normalize query text for comparison"""
|
||||
import re
|
||||
normalized = query.lower().strip()
|
||||
normalized = re.sub(r'[^\w\s]', ' ', normalized)
|
||||
normalized = re.sub(r'\s+', ' ', normalized)
|
||||
return normalized.strip()
|
||||
|
||||
|
||||
def generate_cache_key(query: str) -> str:
|
||||
"""Generate a unique cache key from normalized query"""
|
||||
normalized = normalize_query(query)
|
||||
hash_val = hashlib.sha256(normalized.encode()).hexdigest()[:16]
|
||||
return f"{CACHE_PREFIX}query:{hash_val}"
|
||||
|
||||
|
||||
def embedding_to_bytes(embedding: List[float]) -> bytes:
|
||||
"""Convert embedding list to compact binary format"""
|
||||
return struct.pack(f'{len(embedding)}f', *embedding)
|
||||
|
||||
|
||||
def bytes_to_embedding(data: bytes) -> List[float]:
|
||||
"""Convert binary format back to embedding list"""
|
||||
count = len(data) // 4 # 4 bytes per float
|
||||
return list(struct.unpack(f'{count}f', data))
|
||||
|
||||
|
||||
def cosine_similarity(a: List[float], b: List[float]) -> float:
|
||||
"""Compute cosine similarity between two vectors"""
|
||||
if len(a) != len(b) or len(a) == 0:
|
||||
return 0.0
|
||||
|
||||
a_np = np.array(a)
|
||||
b_np = np.array(b)
|
||||
|
||||
dot_product = np.dot(a_np, b_np)
|
||||
norm_a = np.linalg.norm(a_np)
|
||||
norm_b = np.linalg.norm(b_np)
|
||||
|
||||
if norm_a == 0 or norm_b == 0:
|
||||
return 0.0
|
||||
|
||||
return float(dot_product / (norm_a * norm_b))
|
||||
|
||||
|
||||
def jaccard_similarity(a: str, b: str) -> float:
|
||||
"""Compute Jaccard similarity between two strings (word-level)"""
|
||||
set_a = set(normalize_query(a).split())
|
||||
set_b = set(normalize_query(b).split())
|
||||
|
||||
if not set_a or not set_b:
|
||||
return 0.0
|
||||
|
||||
intersection = len(set_a & set_b)
|
||||
union = len(set_a | set_b)
|
||||
|
||||
return intersection / union if union > 0 else 0.0
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Redis/Valkey Client
|
||||
# =============================================================================
|
||||
|
||||
class ValkeyClient:
|
||||
"""Async Valkey/Redis client wrapper"""
|
||||
|
||||
def __init__(self):
|
||||
self.client: Optional[redis.Redis] = None
|
||||
self.stats = {
|
||||
"hits": 0,
|
||||
"misses": 0,
|
||||
}
|
||||
|
||||
async def connect(self):
|
||||
"""Initialize connection to Valkey"""
|
||||
self.client = redis.Redis(
|
||||
host=VALKEY_HOST,
|
||||
port=VALKEY_PORT,
|
||||
password=VALKEY_PASSWORD,
|
||||
db=VALKEY_DB,
|
||||
decode_responses=False, # We handle encoding ourselves
|
||||
)
|
||||
# Test connection
|
||||
await self.client.ping()
|
||||
print(f"[ValkeyCache] Connected to {VALKEY_HOST}:{VALKEY_PORT}")
|
||||
|
||||
async def disconnect(self):
|
||||
"""Close connection"""
|
||||
if self.client:
|
||||
await self.client.close()
|
||||
print("[ValkeyCache] Disconnected")
|
||||
|
||||
async def lookup(
|
||||
self,
|
||||
query: str,
|
||||
embedding: Optional[List[float]] = None,
|
||||
similarity_threshold: float = SIMILARITY_THRESHOLD,
|
||||
) -> CacheLookupResponse:
|
||||
"""Look up a query in the cache"""
|
||||
start_time = time.time()
|
||||
|
||||
if not self.client:
|
||||
raise HTTPException(status_code=503, detail="Cache not connected")
|
||||
|
||||
normalized = normalize_query(query)
|
||||
|
||||
# 1. Check for exact match first (fastest)
|
||||
exact_key = generate_cache_key(query)
|
||||
exact_match = await self.client.get(exact_key)
|
||||
|
||||
if exact_match:
|
||||
entry = json.loads(exact_match.decode('utf-8'))
|
||||
# Update access time
|
||||
entry['last_accessed'] = int(time.time() * 1000)
|
||||
entry['hit_count'] = entry.get('hit_count', 0) + 1
|
||||
await self.client.setex(exact_key, CACHE_TTL_SECONDS, json.dumps(entry))
|
||||
|
||||
self.stats["hits"] += 1
|
||||
lookup_time = (time.time() - start_time) * 1000
|
||||
|
||||
return CacheLookupResponse(
|
||||
found=True,
|
||||
similarity=1.0,
|
||||
method="exact",
|
||||
lookup_time_ms=lookup_time,
|
||||
entry=entry,
|
||||
)
|
||||
|
||||
# 2. Semantic similarity search
|
||||
best_match = None
|
||||
best_similarity = 0.0
|
||||
match_method = "none"
|
||||
|
||||
# Get all cache keys
|
||||
all_keys = await self.client.keys(f"{CACHE_PREFIX}query:*")
|
||||
|
||||
for key in all_keys:
|
||||
entry_data = await self.client.get(key)
|
||||
if not entry_data:
|
||||
continue
|
||||
|
||||
entry = json.loads(entry_data.decode('utf-8'))
|
||||
|
||||
# Semantic similarity (if embeddings available)
|
||||
if embedding and entry.get('embedding'):
|
||||
stored_embedding = entry['embedding']
|
||||
similarity = cosine_similarity(embedding, stored_embedding)
|
||||
|
||||
if similarity > best_similarity and similarity >= similarity_threshold:
|
||||
best_similarity = similarity
|
||||
best_match = entry
|
||||
match_method = "semantic"
|
||||
|
||||
# Fuzzy text fallback
|
||||
if not best_match:
|
||||
text_similarity = jaccard_similarity(normalized, entry.get('query_normalized', ''))
|
||||
if text_similarity > best_similarity and text_similarity >= 0.85:
|
||||
best_similarity = text_similarity
|
||||
best_match = entry
|
||||
match_method = "fuzzy"
|
||||
|
||||
lookup_time = (time.time() - start_time) * 1000
|
||||
|
||||
if best_match:
|
||||
# Update stats
|
||||
best_match['last_accessed'] = int(time.time() * 1000)
|
||||
best_match['hit_count'] = best_match.get('hit_count', 0) + 1
|
||||
match_key = generate_cache_key(best_match['query'])
|
||||
await self.client.setex(match_key, CACHE_TTL_SECONDS, json.dumps(best_match))
|
||||
|
||||
self.stats["hits"] += 1
|
||||
|
||||
# Don't send embedding back to client (too large)
|
||||
return_entry = {k: v for k, v in best_match.items() if k != 'embedding'}
|
||||
|
||||
return CacheLookupResponse(
|
||||
found=True,
|
||||
similarity=best_similarity,
|
||||
method=match_method,
|
||||
lookup_time_ms=lookup_time,
|
||||
entry=return_entry,
|
||||
)
|
||||
|
||||
self.stats["misses"] += 1
|
||||
|
||||
return CacheLookupResponse(
|
||||
found=False,
|
||||
similarity=best_similarity,
|
||||
method="none",
|
||||
lookup_time_ms=lookup_time,
|
||||
)
|
||||
|
||||
async def store(
|
||||
self,
|
||||
query: str,
|
||||
embedding: Optional[List[float]],
|
||||
response: CachedResponse,
|
||||
language: str = "nl",
|
||||
model: str = "unknown",
|
||||
) -> str:
|
||||
"""Store a query/response pair in the cache"""
|
||||
if not self.client:
|
||||
raise HTTPException(status_code=503, detail="Cache not connected")
|
||||
|
||||
cache_key = generate_cache_key(query)
|
||||
timestamp = int(time.time() * 1000)
|
||||
|
||||
entry = {
|
||||
"id": cache_key,
|
||||
"query": query,
|
||||
"query_normalized": normalize_query(query),
|
||||
"embedding": embedding,
|
||||
"response": response.model_dump(),
|
||||
"timestamp": timestamp,
|
||||
"last_accessed": timestamp,
|
||||
"hit_count": 0,
|
||||
"language": language,
|
||||
"model": model,
|
||||
}
|
||||
|
||||
await self.client.setex(
|
||||
cache_key,
|
||||
CACHE_TTL_SECONDS,
|
||||
json.dumps(entry),
|
||||
)
|
||||
|
||||
# Enforce max entries (simple LRU)
|
||||
await self._enforce_max_entries()
|
||||
|
||||
print(f"[ValkeyCache] Stored: {query[:50]}...")
|
||||
return cache_key
|
||||
|
||||
async def _enforce_max_entries(self):
|
||||
"""Remove oldest entries if over limit"""
|
||||
all_keys = await self.client.keys(f"{CACHE_PREFIX}query:*")
|
||||
|
||||
if len(all_keys) <= MAX_CACHE_ENTRIES:
|
||||
return
|
||||
|
||||
# Get all entries with timestamps
|
||||
entries = []
|
||||
for key in all_keys:
|
||||
entry_data = await self.client.get(key)
|
||||
if entry_data:
|
||||
entry = json.loads(entry_data.decode('utf-8'))
|
||||
entries.append({
|
||||
"key": key,
|
||||
"last_accessed": entry.get("last_accessed", 0),
|
||||
"hit_count": entry.get("hit_count", 0),
|
||||
})
|
||||
|
||||
# Sort by LRU score (recent access + hit count)
|
||||
entries.sort(key=lambda x: x["last_accessed"] + x["hit_count"] * 1000)
|
||||
|
||||
# Remove oldest
|
||||
to_remove = len(entries) - MAX_CACHE_ENTRIES
|
||||
for entry in entries[:to_remove]:
|
||||
await self.client.delete(entry["key"])
|
||||
|
||||
print(f"[ValkeyCache] Evicted {to_remove} entries")
|
||||
|
||||
async def clear(self):
|
||||
"""Clear all cache entries"""
|
||||
if not self.client:
|
||||
raise HTTPException(status_code=503, detail="Cache not connected")
|
||||
|
||||
all_keys = await self.client.keys(f"{CACHE_PREFIX}*")
|
||||
if all_keys:
|
||||
await self.client.delete(*all_keys)
|
||||
|
||||
self.stats = {"hits": 0, "misses": 0}
|
||||
print("[ValkeyCache] Cache cleared")
|
||||
|
||||
async def get_stats(self) -> CacheStats:
|
||||
"""Get cache statistics"""
|
||||
if not self.client:
|
||||
raise HTTPException(status_code=503, detail="Cache not connected")
|
||||
|
||||
all_keys = await self.client.keys(f"{CACHE_PREFIX}query:*")
|
||||
|
||||
total_size = 0
|
||||
oldest = None
|
||||
newest = None
|
||||
|
||||
for key in all_keys:
|
||||
entry_data = await self.client.get(key)
|
||||
if entry_data:
|
||||
total_size += len(entry_data)
|
||||
entry = json.loads(entry_data.decode('utf-8'))
|
||||
timestamp = entry.get("timestamp", 0)
|
||||
|
||||
if oldest is None or timestamp < oldest:
|
||||
oldest = timestamp
|
||||
if newest is None or timestamp > newest:
|
||||
newest = timestamp
|
||||
|
||||
total = self.stats["hits"] + self.stats["misses"]
|
||||
hit_rate = self.stats["hits"] / total if total > 0 else 0.0
|
||||
|
||||
return CacheStats(
|
||||
total_entries=len(all_keys),
|
||||
total_hits=self.stats["hits"],
|
||||
total_misses=self.stats["misses"],
|
||||
hit_rate=hit_rate,
|
||||
storage_used_bytes=total_size,
|
||||
oldest_entry=oldest,
|
||||
newest_entry=newest,
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# FastAPI Application
|
||||
# =============================================================================
|
||||
|
||||
valkey_client = ValkeyClient()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def lifespan(app: FastAPI):
|
||||
"""Startup and shutdown events"""
|
||||
# Startup
|
||||
try:
|
||||
await valkey_client.connect()
|
||||
except Exception as e:
|
||||
print(f"[ValkeyCache] WARNING: Could not connect to Valkey: {e}")
|
||||
print("[ValkeyCache] Service will run without cache persistence")
|
||||
|
||||
yield
|
||||
|
||||
# Shutdown
|
||||
await valkey_client.disconnect()
|
||||
|
||||
|
||||
app = FastAPI(
|
||||
title="GLAM Semantic Cache API",
|
||||
description="Shared semantic cache backend for RAG query responses",
|
||||
version="1.0.0",
|
||||
lifespan=lifespan,
|
||||
)
|
||||
|
||||
# CORS configuration
|
||||
app.add_middleware(
|
||||
CORSMiddleware,
|
||||
allow_origins=[
|
||||
"http://localhost:5173",
|
||||
"http://localhost:5174",
|
||||
"https://bronhouder.nl",
|
||||
"https://www.bronhouder.nl",
|
||||
],
|
||||
allow_credentials=True,
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# API Endpoints
|
||||
# =============================================================================
|
||||
|
||||
@app.get("/health")
|
||||
async def health_check():
|
||||
"""Health check endpoint"""
|
||||
connected = valkey_client.client is not None
|
||||
try:
|
||||
if connected:
|
||||
await valkey_client.client.ping()
|
||||
except Exception:
|
||||
connected = False
|
||||
|
||||
return {
|
||||
"status": "healthy" if connected else "degraded",
|
||||
"valkey_connected": connected,
|
||||
"config": {
|
||||
"host": VALKEY_HOST,
|
||||
"port": VALKEY_PORT,
|
||||
"ttl_seconds": CACHE_TTL_SECONDS,
|
||||
"max_entries": MAX_CACHE_ENTRIES,
|
||||
"similarity_threshold": SIMILARITY_THRESHOLD,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@app.post("/cache/lookup", response_model=CacheLookupResponse)
|
||||
async def cache_lookup(request: CacheLookupRequest):
|
||||
"""
|
||||
Look up a query in the shared cache.
|
||||
|
||||
Returns the most similar cached response if above the similarity threshold.
|
||||
Supports both semantic (embedding) and fuzzy (text) matching.
|
||||
"""
|
||||
threshold = request.similarity_threshold or SIMILARITY_THRESHOLD
|
||||
|
||||
return await valkey_client.lookup(
|
||||
query=request.query,
|
||||
embedding=request.embedding,
|
||||
similarity_threshold=threshold,
|
||||
)
|
||||
|
||||
|
||||
@app.post("/cache/store")
|
||||
async def cache_store(request: CacheStoreRequest):
|
||||
"""
|
||||
Store a query/response pair in the shared cache.
|
||||
|
||||
The entry will be available to all users for semantic matching.
|
||||
"""
|
||||
cache_key = await valkey_client.store(
|
||||
query=request.query,
|
||||
embedding=request.embedding,
|
||||
response=request.response,
|
||||
language=request.language,
|
||||
model=request.model,
|
||||
)
|
||||
|
||||
return {
|
||||
"success": True,
|
||||
"cache_key": cache_key,
|
||||
"ttl_seconds": CACHE_TTL_SECONDS,
|
||||
}
|
||||
|
||||
|
||||
@app.delete("/cache/clear")
|
||||
async def cache_clear(
|
||||
confirm: bool = Query(False, description="Must be true to clear cache")
|
||||
):
|
||||
"""
|
||||
Clear all entries from the shared cache.
|
||||
|
||||
Requires confirmation parameter to prevent accidental clearing.
|
||||
"""
|
||||
if not confirm:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail="Must pass confirm=true to clear cache"
|
||||
)
|
||||
|
||||
await valkey_client.clear()
|
||||
return {"success": True, "message": "Cache cleared"}
|
||||
|
||||
|
||||
@app.get("/cache/stats", response_model=CacheStats)
|
||||
async def cache_stats():
|
||||
"""
|
||||
Get statistics about the shared cache.
|
||||
|
||||
Returns entry counts, hit rates, and storage usage.
|
||||
"""
|
||||
return await valkey_client.get_stats()
|
||||
|
||||
|
||||
@app.get("/cache/entries")
|
||||
async def cache_entries(
|
||||
limit: int = Query(100, ge=1, le=1000),
|
||||
offset: int = Query(0, ge=0),
|
||||
):
|
||||
"""
|
||||
List cached entries (for debugging/admin).
|
||||
|
||||
Returns entries without embeddings to reduce payload size.
|
||||
"""
|
||||
if not valkey_client.client:
|
||||
raise HTTPException(status_code=503, detail="Cache not connected")
|
||||
|
||||
all_keys = await valkey_client.client.keys(f"{CACHE_PREFIX}query:*")
|
||||
all_keys = sorted(all_keys)[offset:offset + limit]
|
||||
|
||||
entries = []
|
||||
for key in all_keys:
|
||||
entry_data = await valkey_client.client.get(key)
|
||||
if entry_data:
|
||||
entry = json.loads(entry_data.decode('utf-8'))
|
||||
# Remove embedding from response
|
||||
entry.pop('embedding', None)
|
||||
entries.append(entry)
|
||||
|
||||
return {
|
||||
"total": len(all_keys),
|
||||
"offset": offset,
|
||||
"limit": limit,
|
||||
"entries": entries,
|
||||
}
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# Main
|
||||
# =============================================================================
|
||||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run(
|
||||
"main:app",
|
||||
host="0.0.0.0",
|
||||
port=8090,
|
||||
reload=True,
|
||||
)
|
||||
7
backend/valkey/requirements.txt
Normal file
7
backend/valkey/requirements.txt
Normal file
|
|
@ -0,0 +1,7 @@
|
|||
# Valkey Semantic Cache Backend
|
||||
|
||||
fastapi>=0.104.0
|
||||
uvicorn[standard]>=0.24.0
|
||||
redis>=5.0.0
|
||||
numpy>=1.24.0
|
||||
pydantic>=2.0.0
|
||||
|
|
@ -0,0 +1,677 @@
|
|||
{
|
||||
"extraction_metadata": {
|
||||
"source_file": "/Users/kempersc/apps/glam/data/custodian/person/affiliated/parsed/3-october-vereeniging_staff_20251210T155412Z.json",
|
||||
"extraction_date": "2025-12-10T16:00:00Z",
|
||||
"extraction_method": "exa_crawling_exa",
|
||||
"total_profiles_in_source": 20,
|
||||
"profiles_with_linkedin_urls": 19,
|
||||
"profiles_extracted": 18,
|
||||
"profiles_failed": 1,
|
||||
"total_cost_usd": 0.018,
|
||||
"extraction_time_seconds": 180
|
||||
},
|
||||
"linkedin_profiles": {
|
||||
"christel-schollaardt-0a605b7": {
|
||||
"staff_id": "3-october-vereeniging_staff_0001_christel_schollaardt",
|
||||
"name": "Christel Schollaardt",
|
||||
"linkedin_url": "https://www.linkedin.com/in/christel-schollaardt-0a605b7",
|
||||
"headline": "Hoofd Collecties en Onderzoek / Plaatsvervangend directeur at Rijksmuseum Boerhaave",
|
||||
"location": "Leiden, South Holland, Netherlands",
|
||||
"connections": "500 connections • 2,135 followers",
|
||||
"about": "Specialties: - Archivist of 3-October Vereeniging 2025 - Vice president of NatHist 2016-2021 -President of ICOMON 2010 - 2016 -lid van het Bestuur van de sectie Collecties van de NMV 2010- 2016 - Mede- eigenaar van Uitgeverij De Muze 2013-2024 When I was a child, my grandmother used to take me for long walks, naming every plant we came along. I was fascinated by her knowledge, and even considered studying biology, but ended up studying library sciences and later on numismatics. But life has its ways and now I am head of botanical collections at Naturalis since a year. I spent most of my career at Geldmuseum and its predecessor, Royal Coin Cabinet. First as librarian and later on as collection manager and Head of department of Collections and Research. At Naturalis, we preserve largest herbarium collection of Netherlands, over 4 million plants!, most of them dried flat and attached to sheets. Much more unknown is fact that we also preserve a large collection of economical end ethno botanical items. I consider it a challenge to highlight these collections and stress importance in upcoming years.",
|
||||
"experience": [
|
||||
{
|
||||
"title": "Archivaris at 3 October Vereeniging",
|
||||
"company": "3 October Vereeniging",
|
||||
"duration": "Feb 2025 - Present • 9 months",
|
||||
"location": "Leiden, South Holland, Netherlands",
|
||||
"company_details": "51-200 employees • Founded 1886 • Nonprofit • Civic and Social Organizations",
|
||||
"department": "Administrative • Level: Specialist"
|
||||
},
|
||||
{
|
||||
"title": "Hoofd Collecties En Onderzoek Plaatsvervangend Directeur at Rijksmuseum Boerhaave",
|
||||
"company": "Rijksmuseum Boerhaave",
|
||||
"duration": "Jan 2021 - Present • 4 years and 10 months",
|
||||
"location": "Leiden, South Holland, Netherlands",
|
||||
"company_details": "51-200 employees • Founded 1931 • Nonprofit • Museums, Historical Sites, and Zoos"
|
||||
},
|
||||
{
|
||||
"title": "Director at Schollaardt Cultural Logistics",
|
||||
"company": "Schollaardt Cultural Logistics",
|
||||
"duration": "May 2012 - Present • 13 years and 6 months",
|
||||
"location": "Leiden",
|
||||
"department": "C-Suite • Level: Director"
|
||||
},
|
||||
{
|
||||
"title": "Mede-eigenaar at Uitgeverij De Muze",
|
||||
"company": "Uitgeverij De Muze",
|
||||
"duration": "Mar 2013 - Sep 2024 • 11 years and 6 months",
|
||||
"location": "Leiden",
|
||||
"company_details": "Company: 1-10 employees • Partnership • Book and Periodical Publishing"
|
||||
},
|
||||
{
|
||||
"title": "AfdelingsHoofd Collecties at Naturalis Biodiversity Center",
|
||||
"company": "Naturalis Biodiversity Center",
|
||||
"duration": "Mar 2019 - Jan 2021 • 1 year and 10 months",
|
||||
"location": "Leiden, South Holland, Netherlands",
|
||||
"company_details": "Company: 501-1000 employees • Founded 1820 • Nonprofit • Museums, Historical Sites, and Zoos"
|
||||
}
|
||||
],
|
||||
"education": [
|
||||
{
|
||||
"degree": "Management",
|
||||
"institution": "IBO Zeist",
|
||||
"duration": "2002 - 2003 • 1 year"
|
||||
},
|
||||
{
|
||||
"degree": "Leiding geven aan professionals",
|
||||
"institution": "De Baak Noordwijk",
|
||||
"duration": "2010 - 2011 • 1 year"
|
||||
},
|
||||
{
|
||||
"degree": "BS, Bibliotheekwetenschappen",
|
||||
"institution": "Haagse Hogeschool/TH Rijswijk",
|
||||
"duration": "1981 - 1984 • 3 years"
|
||||
}
|
||||
],
|
||||
"skills": ["biology", "collections", "research", "director"],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/D4E03AQEExxa_AyW5UA/profile-displayphoto-shrink_200_200/B4EZWpdSUtHcAY-/0/1742304782202?e=2147483647&v=beta&t=65sjghh9CbEPQYmCsPKyYIoD7wnRhxPhdFKn2jGXZSI"
|
||||
},
|
||||
"lodewijkwisse": {
|
||||
"staff_id": "3-october-vereeniging_staff_0002_lodewijk_wisse",
|
||||
"name": "Lodewijk Wisse",
|
||||
"linkedin_url": "https://www.linkedin.com/in/lodewijkwisse",
|
||||
"headline": "Legal, Finance and Tax at Royal Association of Netherlands Shipowners",
|
||||
"location": "The Randstad, Netherlands",
|
||||
"connections": "500 connections • 2,787 followers",
|
||||
"experience": [
|
||||
{
|
||||
"title": "Senior Beleidsmedewerker Juridisch, Financieel En Fiscaal Ondernemingsklimaat at Koninklijke Vereniging van Nederlandse Reders (KVNR)",
|
||||
"company": "Koninklijke Vereniging van Nederlandse Reders (KVNR)",
|
||||
"duration": "Nov 2016 - Present • 9 years",
|
||||
"location": "Rotterdam",
|
||||
"company_details": "Company: 11-50 employees • Founded 1992 • Nonprofit • Maritime Transportation"
|
||||
},
|
||||
{
|
||||
"title": "Lid Raad Van Commissarissen Member Supervisory Board at Leiden&Partners",
|
||||
"company": "Leiden&Partners",
|
||||
"duration": "Mar 2025 - Present • 8 months",
|
||||
"location": "Leiden, Zuid-Holland, Nederland",
|
||||
"department": "C-Suite • Level: Director"
|
||||
},
|
||||
{
|
||||
"title": "Bestuurslid at 3 October Vereeniging",
|
||||
"company": "3 October Vereeniging",
|
||||
"duration": "May 2021 - Present • 4 years and 6 months",
|
||||
"location": "Leiden, Zuid-Holland, Nederland",
|
||||
"department": "C-Suite • Level: Specialist",
|
||||
"role": "Secretaris en vicevoorzitter"
|
||||
}
|
||||
],
|
||||
"education": [
|
||||
{
|
||||
"degree": "Leergang Vervoerrecht",
|
||||
"institution": "Erasmus Universiteit Rotterdam",
|
||||
"duration": "2020 - 2022 • 2 years"
|
||||
},
|
||||
{
|
||||
"degree": "LL.M., Tax law",
|
||||
"institution": "Universiteit Leiden",
|
||||
"duration": "2000 - 2007 • 7 years"
|
||||
},
|
||||
{
|
||||
"degree": "VWO",
|
||||
"institution": "Cals College",
|
||||
"duration": "1994 - 2000 • 6 years"
|
||||
}
|
||||
],
|
||||
"skills": ["tax", "organiseren", "maritime"],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/D4E03AQHRamyjX_6GOQ/profile-displayphoto-shrink_200_200/profile-displayphoto-shrink_200_200/0/1729525154925?e=2147483647&v=beta&t=nNXxx6hQGYGxuc4dkoY6E6iNKiOxGv-t1ihuFX-aYz4"
|
||||
},
|
||||
"rogier-van-der-sande-57436a1": {
|
||||
"staff_id": "3-october-vereeniging_staff_0003_rogier_van_der_sande",
|
||||
"name": "Rogier van der Sande",
|
||||
"linkedin_url": "https://www.linkedin.com/in/rogier-van-der-sande-57436a1",
|
||||
"headline": "Dijkgraaf Hoogheemraadschap van Rijnland",
|
||||
"location": "Leiden, South Holland, Netherlands",
|
||||
"connections": "500 connections • 4,864 followers",
|
||||
"experience": [
|
||||
{
|
||||
"title": "Voorzitter Bestuur (nevenfunctie) at 3 October Vereeniging",
|
||||
"company": "3 October Vereeniging",
|
||||
"duration": "May 2025 - Present • 6 months",
|
||||
"location": "Leiden, Zuid-Holland, Nederland",
|
||||
"company_details": "Company: 51-200 employees • Founded 1886 • Nonprofit • Civic and Social Organizations"
|
||||
},
|
||||
{
|
||||
"title": "Dijkgraaf at Hoogheemraadschap van Rijnland",
|
||||
"company": "Hoogheemraadschap van Rijnland",
|
||||
"duration": "Sep 2017 - Present • 8 years and 2 months",
|
||||
"location": "Leiden",
|
||||
"company_details": "Company: 501-1000 employees • Founded 1255 • Government Agency • Government Administration",
|
||||
"department": "Other • Level: Specialist"
|
||||
}
|
||||
],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/D4E03AQFmOYb0mMTPDg/profile-displayphoto-shrink_200_200/B4EZTWEPGzHMAc-/0/1738758223028?e=2147483647&v=beta&t=9C5AkDFov0zbUZUwN-EtJDqiGeDl6aSji-m1g7St1Xo"
|
||||
},
|
||||
"annette-los-30330a12": {
|
||||
"staff_id": "3-october-vereeniging_staff_0004_annette_los",
|
||||
"name": "Annette Los",
|
||||
"linkedin_url": "https://www.linkedin.com/in/annette-los-30330a12",
|
||||
"headline": "Cultureel Erfgoed I Publieksgericht I Maatschappelijk betrokken I PR & Communicatie",
|
||||
"location": "Netherlands",
|
||||
"connections": "500 connections • 1,501 followers",
|
||||
"experience": [
|
||||
{
|
||||
"title": "Bestuurslid at Vrienden van de Hooglandsekerk",
|
||||
"company": "Vrienden van de Hooglandsekerk",
|
||||
"duration": "Nov 2021 - Present • 3 years and 11 months",
|
||||
"department": "C-Suite • Level: Specialist"
|
||||
},
|
||||
{
|
||||
"title": "Bestuurslid at 3 October Vereeniging",
|
||||
"company": "3 October Vereeniging",
|
||||
"duration": "May 2016 - Present • 9 years and 5 months",
|
||||
"location": "Leiden",
|
||||
"company_details": "Company: 51-200 employees • Founded 1886 • Nonprofit • Civic and Social Organizations",
|
||||
"role": "Voorzitter PR-commissie"
|
||||
},
|
||||
{
|
||||
"title": "Bestuurslid at Stichting Vrienden van de Leidse Hortus",
|
||||
"company": "Stichting Vrienden van de Leidse Hortus",
|
||||
"duration": "Sep 2021 - Present • 4 years and 1 month",
|
||||
"location": "Leiden",
|
||||
"department": "C-Suite • Level: Specialist"
|
||||
}
|
||||
],
|
||||
"education": [
|
||||
{
|
||||
"degree": "drs., Niet-westerse sociologie",
|
||||
"institution": "Leiden University",
|
||||
"duration": "1981 - 1989 • 8 years"
|
||||
},
|
||||
{
|
||||
"degree": "at Reinwardt Academie",
|
||||
"institution": "Reinwardt Academie",
|
||||
"duration": "1978 - 1981 • 3 years"
|
||||
},
|
||||
{
|
||||
"degree": "at Vrije Hogeschool",
|
||||
"institution": "Vrije Hogeschool",
|
||||
"duration": "1977 - 1977"
|
||||
},
|
||||
{
|
||||
"degree": "Atheneum A",
|
||||
"institution": "Rijnlands Lyceum Oegstgeest",
|
||||
"duration": "1970 - 1977 • 7 years"
|
||||
}
|
||||
],
|
||||
"skills": ["partnerships", "pr"],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/C5603AQEuU-Lh8d-14w/profile-displayphoto-shrink_200_200/profile-displayphoto-shrink_200_200/0/1517785609104?e=2147483647&v=beta&t=Wm03BfAXMHEQbdVYMei8OPRcGqCwHKRS7c2zy2xB8tI"
|
||||
},
|
||||
"guido-marchena": {
|
||||
"staff_id": "3-october-vereeniging_staff_0005_guido_marchena",
|
||||
"name": "Guido Marchena",
|
||||
"linkedin_url": "https://www.linkedin.com/in/guido-marchena",
|
||||
"headline": "Adviseur zorg, culturele en creatieve sector | spreker | musicus",
|
||||
"location": "Leiden, South Holland, Netherlands",
|
||||
"connections": "500 connections • 50 followers",
|
||||
"about": "Directeur van Cultuurfonds Leiden, beter bekend als de cultuurmakelaar. Guido komt zelf uit de culturele sector. Na zijn studies BA Film- en Literatuurwetenschap en opleiding tot dirigent aan het Conservatorium van Amsterdam is hij aan de slag gegaan als maestro. Hij dirigeert symfonieorkest, big band en koor. Vanuit zijn bedrijf GM Creative Productions & Consultancy is hij uitgebreid naar creatieve productie met als resultaat onder andere het Leids Grachtenconcert en een opening van de Erasmus Universiteit Eurekaweek in Ahoy. Daarnaast is hij de zanger van zijn jazzcombo de Guido Marchena Swing Society en pianist/vocalist in Upright Pianos. Vanuit zijn brede creatieve en bestuurlijke ervaring geeft hij media- en PR-advies/training. Daarnaast is hij regelmatig te zien als presentator of moderator bij symposia, debatten, festivals etc en geeft hij op geestige wijze training in etiquette en protocol.",
|
||||
"experience": [
|
||||
{
|
||||
"title": "Independent Professional at Self-employed",
|
||||
"company": "Self-employed",
|
||||
"duration": "Jun 2011 - Present • 14 years and 5 months",
|
||||
"department": "Other • Level: Specialist"
|
||||
},
|
||||
{
|
||||
"title": "Bestuurslid at 3 October Vereeniging",
|
||||
"company": "3 October Vereeniging",
|
||||
"duration": "May 2017 - Present • 8 years and 6 months",
|
||||
"location": "Leiden, Provincie Zuid-Holland, Nederland",
|
||||
"company_details": "Company: 51-200 employees • Founded 1886 • Nonprofit • Civic and Social Organizations",
|
||||
"role": "Vice voorzitter Commissie Herdenkingen Vice voorzitter Commissie Evenementen"
|
||||
}
|
||||
],
|
||||
"education": [
|
||||
{
|
||||
"degree": "Bachelor of Fine Arts - BFA, Conducting",
|
||||
"institution": "Amsterdamse Hogeschool voor de Kunsten",
|
||||
"duration": "2014 - 2018 • 4 years"
|
||||
},
|
||||
{
|
||||
"degree": "Bachelor of Arts (BA), Film- and Literature Studies",
|
||||
"institution": "Leiden University",
|
||||
"duration": "2008 - 2012 • 4 years"
|
||||
}
|
||||
],
|
||||
"skills": ["film", "creative", "consultancy", "swing", "pianist", "vocalist", "pr", "training", "festivals", "producer"],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/C4D03AQGfbtZ9b0KIDg/profile-displayphoto-shrink_200_200/profile-displayphoto-shrink_200_200/0/1610980422630?e=2147483647&v=beta&t=hgNd3FwzX2WXFWrQMFT37cKPTqJjviyG3dvi19Nxdvo"
|
||||
},
|
||||
"elsschipper": {
|
||||
"staff_id": "3-october-vereeniging_staff_0006_els_schipper",
|
||||
"name": "Els Schipper",
|
||||
"linkedin_url": "https://www.linkedin.com/in/elsschipper",
|
||||
"headline": "Ondernemer * dagvoorzitter * creatief producent * kwartiermaker * tekstschrijver * cultureel programmeur",
|
||||
"location": "Alkmaar, North Holland, Netherlands",
|
||||
"connections": "500 connections • 1,093 followers",
|
||||
"experience": [
|
||||
{
|
||||
"title": "Ondernemer Dagvoorzitter Creatief Producent En Programmeur Tekstschrijver at Els Schipper",
|
||||
"company": "Els Schipper",
|
||||
"duration": "Jan 2016 - Present • 9 years and 9 months",
|
||||
"location": "Alkmaar Area, Netherlands"
|
||||
}
|
||||
],
|
||||
"education": [
|
||||
{
|
||||
"degree": "BSc, Communicatiewetenschap",
|
||||
"institution": "University of Amsterdam",
|
||||
"duration": "2000 - 2008 • 8 years"
|
||||
}
|
||||
],
|
||||
"skills": ["social media", "special effects", "sociale media", "creatief", "marketing", "less", "murals", "production", "tennis", "sport", "organiseren", "seo", "twitter", "facebook", "wifi"],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/D4E03AQGANjOu7dnOAg/profile-displayphoto-shrink_200_200/B4EZTQvkd2GwAg-/0/1738668919302?e=2147483647&v=beta&t=TDRLuVzJADu5_4Hz7Zc5wmzgiUSqFIjmQ-t-4d9Kjoo"
|
||||
},
|
||||
"greet-spetter-verhoogt-79317a237": {
|
||||
"staff_id": "3-october-vereeniging_staff_0007_greet_spetter_verhoogt",
|
||||
"name": "Greet Spetter Verhoogt",
|
||||
"linkedin_url": "https://www.linkedin.com/in/greet-spetter-verhoogt-79317a237",
|
||||
"location": "Leiden, South Holland, Netherlands",
|
||||
"connections": "147 connections • 149 followers",
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/D4E03AQFD6YqOwRKoIQ/profile-displayphoto-shrink_200_200/profile-displayphoto-shrink_200_200/0/1672431283871?e=2147483647&v=beta&t=LB4PkssEZusPhoyQGRvQh63oZbZgAO1urY2fB0dnkbw"
|
||||
},
|
||||
"zjerom": {
|
||||
"staff_id": "3-october-vereeniging_staff_0008_jeroen_weijermars",
|
||||
"name": "Jeroen Weijermars",
|
||||
"linkedin_url": "https://www.linkedin.com/in/zjerom",
|
||||
"headline": "Docent sportmarketing, -management en -beleid. (Sportkunde), Haagse Hogeschool",
|
||||
"location": "Oegstgeest, South Holland, Netherlands",
|
||||
"connections": "500 connections • 1,731 followers",
|
||||
"about": "Gepassioneerd door de kracht van sport werd mijn hobby mijn professie. Inmiddels houd ik mij al sinds vorige eeuw bezig met sportmarketing, sportmanagement en sportbeleid. Voor een belangrijk deel werk ik als docent en ben ik verbonden aan de opleiding Sportkunde van de Haagse Hogeschool. Als 'Chef de mission' werk ik aan de missie van de Stichting Sport in Beeld en organiseer ik breedte sportevenementen zoals de Bevrijdingsvuurestafette Avondvierdaagse, Zwemvierdaagse, de Ladiesrun en de Drijvende IJsbaan in Leiden.",
|
||||
"experience": [
|
||||
{
|
||||
"title": "Owner at V.O.F. de Kemp",
|
||||
"company": "V.O.F. de Kemp",
|
||||
"duration": "Jan 2022 - Present • 3 years and 9 months",
|
||||
"location": "Oegstgeest, South Holland, Netherlands",
|
||||
"department": "C-Suite • Level: Owner"
|
||||
},
|
||||
{
|
||||
"title": "Commissaris Optocht- Taptoecommissie at 3 October Vereeniging",
|
||||
"company": "3 October Vereeniging",
|
||||
"duration": "Aug 2021 - Present • 4 years and 2 months",
|
||||
"location": "Leiden, Zuid-Holland, Nederland",
|
||||
"company_details": "Company: 51-200 employees • Founded 1886 • Nonprofit • Civic and Social Organizations"
|
||||
}
|
||||
],
|
||||
"education": [
|
||||
{
|
||||
"degree": "Master of Business Administration (M.B.A.), Master of Business Administration (MBA), Sportmanagement",
|
||||
"institution": "Sport Management Instituut - Wagner Group",
|
||||
"duration": "2012 - 2014 • 2 years"
|
||||
},
|
||||
{
|
||||
"degree": "Basisopleiding HBO-Docent, Higher Education/Higher Education Administration",
|
||||
"institution": "Vrije Universiteit Amsterdam",
|
||||
"duration": "2011 - 2012 • 1 year"
|
||||
}
|
||||
],
|
||||
"skills": ["sport", "sales", "marketing", "management", "sales manager"],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/C4E03AQFbZnoR2DdgDQ/profile-displayphoto-shrink_200_200/profile-displayphoto-shrink_200_200/0/1653400711684?e=2147483647&v=beta&t=YLZZdKKzGPNYYazlEGqu4HpjAAtN8pmS0TNkLo0xyQ0"
|
||||
},
|
||||
"niggebrugge": {
|
||||
"staff_id": "3-october-vereeniging_staff_0009_arthur_h_p_niggebrugge",
|
||||
"name": "Arthur H.P. Niggebrugge",
|
||||
"linkedin_url": "https://www.linkedin.com/in/niggebrugge",
|
||||
"headline": "MD PhD Chirurg (n.p.) LKol (b.d.)",
|
||||
"location": "The Randstad, Netherlands",
|
||||
"connections": "500 connections • 2,220 followers",
|
||||
"about": "Specialties: Hip fracture surgery Trauma surgery General…",
|
||||
"experience": [
|
||||
{
|
||||
"title": "Commissaris 3 October Vereeniging at 3 October Vereeniging",
|
||||
"company": "3 October Vereeniging",
|
||||
"duration": "Jan 2016 - Present • 9 years and 10 months",
|
||||
"location": "Leiden",
|
||||
"company_details": "Company: 51-200 employees • Founded 1886 • Nonprofit • Civic and Social Organizations",
|
||||
"role": "Commissaris 3 October Vereeniging, i.h.b. Haring & Wittebrood commissie, speciaal belast met onderhouden relaties Ambassades en Militaire Missies"
|
||||
}
|
||||
],
|
||||
"education": [
|
||||
{
|
||||
"degree": "PhD, Experimental Studies Closure Abdominal Wall",
|
||||
"institution": "Leiden University",
|
||||
"duration": "1995 - 1999 • 4 years"
|
||||
},
|
||||
{
|
||||
"degree": "MD, medicine",
|
||||
"institution": "Leiden University",
|
||||
"duration": "1982 - 1989 • 7 years"
|
||||
}
|
||||
],
|
||||
"skills": ["surgery", "trauma", "p"],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/D4E03AQEPWOTlRNBvMg/profile-displayphoto-shrink_200_200/B4EZaapYd2GYAc-/0/1746351262020?e=2147483647&v=beta&t=4V3zd_SddmbpJqiB306oQN1f1lrFnUAGUmvoD3BVUA0"
|
||||
},
|
||||
"alexandervannimwegen": {
|
||||
"staff_id": "3-october-vereeniging_staff_0010_alexander_van_nimwegen",
|
||||
"name": "Alexander Van Nimwegen",
|
||||
"linkedin_url": "https://www.linkedin.com/in/alexandervannimwegen",
|
||||
"location": "Leiden, South Holland, Netherlands",
|
||||
"connections": "500 connections • 72 followers",
|
||||
"experience": [
|
||||
{
|
||||
"title": "Co-Founder at FashionQlub",
|
||||
"company": "FashionQlub",
|
||||
"department": "C-Suite • Level: Founder"
|
||||
}
|
||||
],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/C5603AQGLoRRdvpBqMA/profile-displayphoto-shrink_200_200/profile-displayphoto-shrink_200_200/0/1516276766786?e=2147483647&v=beta&t=H5-9at2wwXJ2G9L8mw6edilI13p7MbyOGYqvhb3r1_8"
|
||||
},
|
||||
"bernadette-drenth-46369712": {
|
||||
"staff_id": "3-october-vereeniging_staff_0011_bernadette_drenth",
|
||||
"name": "Bernadette Drenth",
|
||||
"linkedin_url": "https://www.linkedin.com/in/bernadette-drenth-46369712",
|
||||
"headline": "designer at TEX & uitleg",
|
||||
"location": "Leiden, South Holland, Netherlands",
|
||||
"connections": "304 connections • 307 followers",
|
||||
"about": "Specialties: textielvormgeving kostuums wereable art decor theaterontwerp lessen en workshops bedrijfspromotie",
|
||||
"experience": [
|
||||
{
|
||||
"title": "Designer at TEX & uitleg",
|
||||
"company": "TEX & uitleg",
|
||||
"department": "Design • Level: Specialist"
|
||||
},
|
||||
{
|
||||
"title": "Optochtcie at 3 october vereeniging",
|
||||
"company": "3 October Vereeniging",
|
||||
"duration": "Jan 2000 - Present • 25 years and 1 month"
|
||||
}
|
||||
],
|
||||
"education": [
|
||||
{
|
||||
"degree": "1e graad TEX en kunstgeschiedenis, textile design",
|
||||
"institution": "Amsterdamse Hogeschool voor de Kunsten",
|
||||
"duration": "1987 - 1991 • 4 years"
|
||||
}
|
||||
],
|
||||
"skills": ["workshops"],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/C4D03AQEJQDTDosHoiQ/profile-displayphoto-shrink_200_200/profile-displayphoto-shrink_200_200/0/1516429326152?e=2147483647&v=beta&t=O7IxdQt_mtcByWNnnoJCdAc1w6lFHx2YfoggR9Y4XmY"
|
||||
},
|
||||
"jurgen-van-der-velden-19101434": {
|
||||
"staff_id": "3-october-vereeniging_staff_0013_jurgen_van_der_velden",
|
||||
"name": "Jurgen van der Velden",
|
||||
"linkedin_url": "https://www.linkedin.com/in/jurgen-van-der-velden-19101434",
|
||||
"headline": "Consultant Radiologist / Interventional Radiologist",
|
||||
"location": "The Randstad, Netherlands",
|
||||
"connections": "500 connections • 501 followers",
|
||||
"about": "Consulting Radiologist / Interventional Radiologist Founder GRS Managing Consultant…",
|
||||
"experience": [
|
||||
{
|
||||
"title": "Founder at GRS",
|
||||
"company": "GRS",
|
||||
"duration": "Jan 2006 - Present • 19 years and 10 months",
|
||||
"location": "Rotterdam Area, Netherlands",
|
||||
"department": "C-Suite • Level: Founder"
|
||||
},
|
||||
{
|
||||
"title": "Consultant Radiologist Interventional Radiologist at St. Franciscus Gasthuis",
|
||||
"company": "St. Franciscus Gasthuis",
|
||||
"duration": "Feb 1999 - Present • 26 years and 9 months",
|
||||
"location": "Rotterdam, The Netherlands",
|
||||
"company_details": "Company: 1001-5000 employees • Founded 1892 • Privately Held • Hospitals and Health Care",
|
||||
"department": "Medical • Level: Specialist"
|
||||
},
|
||||
{
|
||||
"title": "Secretaris at 3 October Vereeniging",
|
||||
"company": "3 October Vereeniging",
|
||||
"duration": "Jan 2007 - Present • 18 years and 9 months",
|
||||
"location": "Leiden, The Netherlands",
|
||||
"company_details": "www.3october.nl",
|
||||
"department": "Administrative • Level: Specialist"
|
||||
}
|
||||
],
|
||||
"education": [
|
||||
{
|
||||
"degree": "PhD, Thesis",
|
||||
"institution": "Erasmus Universiteit Rotterdam / Erasmus University Rotterdam",
|
||||
"duration": "2011 - 2012 • 1 year"
|
||||
},
|
||||
{
|
||||
"degree": "Diagnostic Radiology Residency Program",
|
||||
"institution": "Erasmus University Rotterdam",
|
||||
"duration": "1992 - 1997 • 5 years"
|
||||
},
|
||||
{
|
||||
"degree": "Medical Doctor",
|
||||
"institution": "Leiden University",
|
||||
"duration": "1982 - 1990 • 8 years"
|
||||
}
|
||||
],
|
||||
"skills": ["consulting"],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/C5603AQGUnJ_WYU6EPw/profile-displayphoto-shrink_200_200/profile-displayphoto-shrink_200_200/0/1516549510594?e=2147483647&v=beta&t=vhPDB3G4fGydP__6u5N9FfNgo6thiFMtrxyPEKchZ9A"
|
||||
},
|
||||
"niekhoogwout": {
|
||||
"staff_id": "3-october-vereeniging_staff_0014_niek_hoogwout",
|
||||
"name": "Niek Hoogwout",
|
||||
"linkedin_url": "https://www.linkedin.com/in/niekhoogwout",
|
||||
"headline": "Advocaat | Aanbestedingsrecht | Bouwrecht | Partner La Gro",
|
||||
"location": "The Hague, South Holland, Netherlands",
|
||||
"connections": "500 connections • 1,432 followers",
|
||||
"about": "LaGro helpt architecten, aannemers ontwikkelaars en overheden bij de aanbesteding, uitleg en uitvoering van o.a. bouwcontracten. Bijvoorbeeld: - hoe moet je een overheidsopdracht aanbesteden - wat kan je doen tegen een afwijzing bij een aanbesteding - hoe en wanneer meld je meerwerk - wat te doen als een werk niet tijdig en/of niet deugdelijk wordt uitgevoerd of opgeleverd - wat kan je doen als een ander je ontwerp imiteert - wat je kan doen als een derde je ontwerp nabouwt zonder dat je het wist - kan ik aanspraak maken op honorarium als de opdrachtgever de opdracht stopzet? Cliënten waarderen onze praktische insteek, snelle en efficiënte werkwijze, deskundigheid, daadkracht, doortastendheid en vasthoudendheid. In mijn vrije tijd loop ik graag (net zo) hard. PB marathon Hamburg 2017 in 2:49:46.",
|
||||
"experience": [
|
||||
{
|
||||
"title": "Juridisch Expert Aanbestedingsrecht at Ombudsman Aanbesteding (OMA)",
|
||||
"company": "Ombudsman Aanbesteding (OMA)",
|
||||
"duration": "Jun 2022 - Present • 3 years and 4 months",
|
||||
"location": "Leiden, Zuid-Holland, Nederland",
|
||||
"company_details": "Company: 1-10 employees • Founded 2021 • Nonprofit • Public Policy Offices"
|
||||
},
|
||||
{
|
||||
"title": "Voorzitter Departement Rijnsteden at Lighthouse Club Nederland",
|
||||
"company": "Lighthouse Club Nederland",
|
||||
"duration": "Apr 2020 - Present • 5 years and 6 months",
|
||||
"location": "Noordwijk aan Zee, Zuid-Holland, Nederland"
|
||||
},
|
||||
{
|
||||
"title": "Commissie PR En Ledenwerving at 3-Octobervereniging Leiden",
|
||||
"company": "3-Octobervereniging Leiden",
|
||||
"duration": "Apr 2009 - Present • 16 years and 6 months",
|
||||
"location": "Leiden",
|
||||
"role": "Als commissaris van de 3-October Vereeniging ben ik betrokken bij de PR rondom het grootste en mooiste feest van Leiden e.o.. De commissie PR gaat, onder andere, over de uitingen naar buiten, de ledenpas, ledenwerving, ledenacties, sponsoring en persmededelingen. En natuurlijk de verkoop van merchandise in de Rjdende Geus en via de marktkramen van de 3-October Vereeniging!"
|
||||
}
|
||||
],
|
||||
"education": [
|
||||
{
|
||||
"degree": "mr, Nederlands recht",
|
||||
"institution": "Erasmus Universiteit Rotterdam",
|
||||
"duration": "1987 - 1994 • 7 years"
|
||||
}
|
||||
],
|
||||
"skills": ["pr"],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/C4E03AQGoTLPmi1eHwA/profile-displayphoto-shrink_200_200/profile-displayphoto-shrink_200_200/0/1654677775271?e=2147483647&v=beta&t=_wss54gbgZxVyRRf5ZSr5cCo6j0GpEeUMsAASWtenRU"
|
||||
},
|
||||
"ronny-schuijt-msc-ra-8a5a1a21": {
|
||||
"staff_id": "3-october-vereeniging_staff_0015_ronny_schuijt_msc_ra",
|
||||
"name": "Ronny Schuijt MSc",
|
||||
"linkedin_url": "https://www.linkedin.com/in/ronny-schuijt-msc-ra-8a5a1a21",
|
||||
"headline": "Managing Partner / Registeraccountant / ETL Assurance & Overheidsaccountants",
|
||||
"location": "Amsterdam, North Holland, Netherlands",
|
||||
"connections": "500 connections • 2,000 followers",
|
||||
"experience": [
|
||||
{
|
||||
"title": "Penningmeester Stichting Alkmaar ONTZET Projecten at Stichting Alkmaar ONTZET projecten",
|
||||
"company": "Stichting Alkmaar ONTZET projecten",
|
||||
"duration": "Feb 2023 - Present • 2 years and 9 months",
|
||||
"location": "Alkmaar, Noord-Holland, Nederland",
|
||||
"department": "Finance & Accounting • Level: Specialist"
|
||||
},
|
||||
{
|
||||
"title": "Managing Partner Registeraccountant ETL Assurance Overheidsaccountants at ETL Assurance & Overheidsaccountants",
|
||||
"company": "ETL Assurance & Overheidsaccountants",
|
||||
"duration": "Oct 2022 - Present • 7 months",
|
||||
"location": "Haarlem, North Holland, Netherlands",
|
||||
"company_details": "Company: 11-50 employees • Founded 2022 • Privately Held • Financial Services"
|
||||
},
|
||||
{
|
||||
"title": "Penningmeester 8 October Vereeniging at 8 October Vereeniging",
|
||||
"company": "8 October Vereeniging",
|
||||
"duration": "May 2021 - Present • 2 years",
|
||||
"location": "De 8 October Vereeniging houdt de Alkmaarse geschiedenis levend 8 October Vereeniging zet zich met hart en ziel in voor de viering en herdenking van Alkmaar Ontzet. Jaarlijks staan we stil bij het bijzondere feit dat in 1573 de Victorie in Alkmaar begon. Toen moest het Spaanse leger het beleg van de stad opgeven nadat de dijken rond Alkmaar door waren gestoken. Daarmee was Alkmaar een van de eerste Hollandse steden die de Spaanse belegering met succes had weerstaan. En dat is ieder jaar reden tot bezinning en feest! Tijdens de 8 October Viering knapt de stad uit z'n voegen van de feestelijkheden: de lampionoptocht, jeugdoptocht, grote middagoptocht, zuurkoolmaaltijd, ringsteken, vletten racen, historische kermis en nog veel meer. Als vereniging staan we samen met alle Alkmaarders stil bij de betekenis van deze historische gebeurtenis en vieren de vrijheid!"
|
||||
}
|
||||
],
|
||||
"education": [
|
||||
{
|
||||
"degree": "MSc RA, Auditing",
|
||||
"institution": "Nyenrode Business Universiteit",
|
||||
"duration": "1999 - 2006 • 7 years"
|
||||
}
|
||||
],
|
||||
"skills": ["director", "accountants", "finance", "less", "management", "etl", "assurance", "ict", "organized", "spark", "regulations", "b2c", "commerce", "selling", "logistics", "portals", "road", "non profit", "corporate finance", "customer service"],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/C5603AQFqogPfTcUksA/profile-displayphoto-shrink_200_200/profile-displayphoto-shrink_200_200/0/1574692295197?e=2147483647&v=beta&t=qY2luRLcqnpw1ODpmWFAvzVV1PIWa-5FkBbG-izdvjw"
|
||||
},
|
||||
"durmus-dogan-b3124133": {
|
||||
"staff_id": "3-october-vereeniging_staff_0016_durmus_dogan",
|
||||
"name": "Durmus Dogan",
|
||||
"linkedin_url": "https://www.linkedin.com/in/durmus-dogan-b3124133",
|
||||
"headline": "Voorzitter / Baskan at Turks-Nederlands Ondernemersvereniging Platform / Hollanda Turk Girisimci Dernekleri Platformu",
|
||||
"location": "The Hague, South Holland, Netherlands",
|
||||
"connections": "500 connections • 1,129 followers",
|
||||
"experience": [
|
||||
{
|
||||
"title": "Lid Ledenraad at Rabobank",
|
||||
"company": "Rabobank",
|
||||
"duration": "Sep 2024 - Present • 1 year and 1 month",
|
||||
"company_details": "Company: 10,001+ employees • Privately Held • Banking"
|
||||
},
|
||||
{
|
||||
"title": "Commissaris Van Haring En Wittebrood at 3 October Vereeniging",
|
||||
"company": "3 October Vereeniging",
|
||||
"duration": "May 2023 - Present • 2 years and 5 months",
|
||||
"location": "Leiden, South Holland, Netherlands"
|
||||
}
|
||||
],
|
||||
"education": [
|
||||
{
|
||||
"degree": "at Grotius Academie",
|
||||
"institution": "Grotius Academie",
|
||||
"duration": "2015 - 2015"
|
||||
}
|
||||
],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/C5603AQFy3lknG6MXkQ/profile-displayphoto-shrink_200_200/profile-displayphoto-shrink_200_200/0/1516526624969?e=2147483647&v=beta&t=I8sSY8-9WlKmiuscaInPuorcJ55WCd1Y3VAJ0ry4WTo"
|
||||
},
|
||||
"eva-mulder-arends-1433495": {
|
||||
"staff_id": "3-october-vereeniging_staff_0017_eva_mulder_arends",
|
||||
"name": "Eva Mulder - Arends",
|
||||
"linkedin_url": "https://www.linkedin.com/in/eva-mulder-arends-1433495",
|
||||
"headline": "Retail - Art | Fashion | Interior",
|
||||
"location": "Netherlands",
|
||||
"connections": "500 connections • 1,126 followers",
|
||||
"experience": [
|
||||
{
|
||||
"title": "Manager at Noort Interieur",
|
||||
"company": "Noort Interieur",
|
||||
"duration": "Jan 2025 - Present • 10 months",
|
||||
"location": "Noordwijk, Zuid-Holland, Nederland",
|
||||
"company_details": "Company: 1-10 employees • Public Company • Interior Design"
|
||||
}
|
||||
],
|
||||
"education": [
|
||||
{
|
||||
"degree": "Ingenieur (Ir.), Technisch Commerciele Confectiekunde Meester Koetsier., Fashion Business",
|
||||
"institution": "Amsterdamse Hogeschool voor de Kunsten",
|
||||
"duration": "1997 - 2002 • 5 years"
|
||||
}
|
||||
],
|
||||
"skills": ["buying", "design", "stores", "accessories", "projects", "retail", "responsible", "knitwear", "kpi", "budget", "sales", "management", "fashion", "wholesale", "director", "creative", "integrity", "collaboration", "innovation", "museums", "partnerships", "leadership", "operations", "commercial", "organizing", "galleries", "curating", "artists", "set design", "fashion buying", "financial management", "contemporary art", "art gallery", "new business opportunities"],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/D4E03AQHFlnrtfwrY9Q/profile-displayphoto-shrink_200_200/profile-displayphoto-shrink_200_200/0/1760267326539?e=2147483647&v=beta&t=4r_wNo3QquZO9PHR_HSyhuzsYMsN0PGNPyz3ClI8HTc"
|
||||
},
|
||||
"ronald-schenk-4373636": {
|
||||
"staff_id": "3-october-vereeniging_staff_0018_ronald_schenk",
|
||||
"name": "Ronald Schenk",
|
||||
"linkedin_url": "https://www.linkedin.com/in/ronald-schenk-4373636",
|
||||
"headline": "Bestuurslid bij Stichting Vrienden van Alecto",
|
||||
"location": "The Hague, South Holland, Netherlands",
|
||||
"connections": "500 connections • 1,104 followers",
|
||||
"experience": [
|
||||
{
|
||||
"title": "ZZP-er at R. Schenk Financiële- & Facilitaire Dienstverlening",
|
||||
"company": "R. Schenk Financiële- & Facilitaire Dienstverlening",
|
||||
"duration": "Jan 2013 - Present • 12 years and 10 months",
|
||||
"location": "Leiderdorp"
|
||||
},
|
||||
{
|
||||
"title": "Zaakvoerder at IBRR | Incasso & Credit management",
|
||||
"company": "IBRR | Incasso & Credit management",
|
||||
"duration": "Oct 2010 - Present • 8 years and 5 months",
|
||||
"location": "Leiden"
|
||||
},
|
||||
{
|
||||
"title": "Commissaris at 3 October Vereeniging",
|
||||
"company": "3 October Vereeniging",
|
||||
"duration": "Oct 2005 - Present • 13 years and 5 months",
|
||||
"location": "Leiden"
|
||||
}
|
||||
],
|
||||
"education": [
|
||||
{
|
||||
"degree": "Rechten / Rechten en rechtswetenschappen",
|
||||
"institution": "Leiden University",
|
||||
"duration": "1999 - 2001 • 2 years"
|
||||
}
|
||||
],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/C4E03AQEBaBpFZMFpXg/profile-displayphoto-shrink_200_200/profile-displayphoto-shrink_200_200/0/1517680448910?e=2147483647&v=beta&t=D_0RnbZ1oS1QuLpBmZbuMHlF2rNzogTnaXCVtaBUTIY"
|
||||
},
|
||||
"eric-paul-sanders-a6817910b": {
|
||||
"staff_id": "3-october-vereeniging_staff_0019_simon_kemper",
|
||||
"name": "Eric-Paul Sanders",
|
||||
"linkedin_url": "https://www.linkedin.com/in/eric-paul-sanders-a6817910b",
|
||||
"headline": "Owner Sandhold / Winemaker",
|
||||
"location": "Leiden, South Holland, Netherlands",
|
||||
"connections": "500 connections • 2,349 followers",
|
||||
"about": "\"Managing the finest wines and creating future…\"",
|
||||
"experience": [
|
||||
{
|
||||
"title": "Owner at Sandhold",
|
||||
"company": "Sandhold",
|
||||
"duration": "Jan 2005 - Present • 20 years and 10 months",
|
||||
"location": "Leiden",
|
||||
"department": "C-Suite • Level: Owner"
|
||||
},
|
||||
{
|
||||
"title": "Bestuurslid at 3 October Vereeniging",
|
||||
"company": "3 October Vereeniging",
|
||||
"duration": "May 2023 - Present • 2 years and 5 months",
|
||||
"location": "Leiden, Zuid-Holland, Nederland"
|
||||
}
|
||||
],
|
||||
"education": [
|
||||
{
|
||||
"degree": "at Hotelschool The Hague",
|
||||
"institution": "Hotelschool The Hague",
|
||||
"duration": "1998 - 2002 • 4 years"
|
||||
}
|
||||
],
|
||||
"profile_image_url": "https://media.licdn.com/dms/image/v2/D4E03AQEvds_J88A4Vg/profile-displayphoto-shrink_200_200/profile-displayphoto-shrink_200_200/0/1742475016344?e=2147483647&v=beta&t=_s1Sb_tJi1qsFK72WmOePqtUNYdob45OSHuAQCAWLsw"
|
||||
}
|
||||
},
|
||||
"failed_profiles": [
|
||||
{
|
||||
"staff_id": "3-october-vereeniging_staff_0012_marloes_elfferich_is_open_to_w",
|
||||
"name": "Marloes Elfferich is open to work",
|
||||
"linkedin_url": "https://www.linkedin.com/in/marloes-elfferich",
|
||||
"error": "Crawling error (502): Request failed with status code 502"
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"total_profiles_attempted": 19,
|
||||
"successful_extractions": 18,
|
||||
"failed_extractions": 1,
|
||||
"success_rate": 94.7,
|
||||
"total_cost_usd": 0.018,
|
||||
"average_cost_per_profile": 0.001,
|
||||
"extraction_date": "2025-12-10T16:00:00Z",
|
||||
"data_quality": "high",
|
||||
"notes": "Successfully extracted comprehensive LinkedIn profile data including work experience, education, skills, and profile images for 18 out of 19 staff members from 3 October Vereeniging. One profile failed due to LinkedIn access restrictions."
|
||||
}
|
||||
}
|
||||
File diff suppressed because one or more lines are too long
Binary file not shown.
|
After Width: | Height: | Size: 4.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 12 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 44 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 39 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 6.6 KiB |
File diff suppressed because one or more lines are too long
Binary file not shown.
|
After Width: | Height: | Size: 3.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 18 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 34 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 30 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 13 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.1 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 12 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.8 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 3.9 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5.5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 2.6 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.7 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.3 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 5 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.4 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 1.2 KiB |
Binary file not shown.
|
After Width: | Height: | Size: 4.6 KiB |
Some files were not shown because too many files have changed in this diff Show more
Loading…
Reference in a new issue