908 lines
38 KiB
Python
908 lines
38 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Generic enrichment script for custodian YAML files.
|
|
|
|
This script can process any country's custodian files and enrich them from
|
|
multiple sources based on available identifiers:
|
|
|
|
1. **Wikidata ID available** → Full Wikidata enrichment (labels, temporal, identifiers, location, etc.)
|
|
2. **ISIL code available** → Resolve to Wikidata via P791, then full enrichment
|
|
3. **Website URL available** → Could use for web scraping/validation (future)
|
|
|
|
Sources for Wikidata ID detection:
|
|
- wikidata_enrichment.wikidata_entity_id
|
|
- identifiers[].identifier_scheme == "Wikidata"
|
|
- original_entry.wikidata_id
|
|
- original_entry.identifiers[].identifier_scheme == "Wikidata"
|
|
|
|
Usage:
|
|
python scripts/enrich_custodians_generic.py [--country XX] [--dry-run] [--limit N] [--force]
|
|
|
|
Examples:
|
|
# Dry run on Japan files (first 10)
|
|
python scripts/enrich_custodians_generic.py --country JP --limit 10 --dry-run
|
|
|
|
# Enrich all Czech files
|
|
python scripts/enrich_custodians_generic.py --country CZ
|
|
|
|
# Force re-enrichment of already enriched files
|
|
python scripts/enrich_custodians_generic.py --country JP --force
|
|
|
|
# Resume from last checkpoint
|
|
python scripts/enrich_custodians_generic.py --country JP --resume
|
|
|
|
Options:
|
|
--country XX Only process files for country code XX (e.g., JP, CZ, NL)
|
|
--dry-run Show what would be enriched without modifying files
|
|
--limit N Process only first N files (for testing)
|
|
--force Re-enrich even if already has wikidata_enrichment
|
|
--resume Resume from last checkpoint
|
|
--verbose Show detailed progress information
|
|
|
|
Environment Variables:
|
|
WIKIDATA_API_TOKEN - Optional OAuth2 token for increased rate limits (5,000 req/hr)
|
|
|
|
See AGENTS.md Rule 5: NEVER Delete Enriched Data - Additive Only
|
|
"""
|
|
|
|
import argparse
|
|
import json
|
|
import logging
|
|
import os
|
|
import sys
|
|
import time
|
|
from dataclasses import dataclass, field
|
|
from datetime import datetime, timezone
|
|
from pathlib import Path
|
|
from typing import Any, Dict, List, Optional, Set
|
|
|
|
import httpx
|
|
import yaml
|
|
|
|
# Set up logging
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format='%(asctime)s - %(levelname)s - %(message)s'
|
|
)
|
|
logger = logging.getLogger(__name__)
|
|
|
|
# Configuration
|
|
WIKIDATA_REST_API = "https://www.wikidata.org/w/rest.php/wikibase/v1"
|
|
WIKIDATA_SPARQL_ENDPOINT = "https://query.wikidata.org/sparql"
|
|
CUSTODIAN_DIR = Path(__file__).parent.parent / "data" / "custodian"
|
|
PROGRESS_FILE = Path(__file__).parent.parent / "data" / "custodian" / ".generic_enrichment_progress.json"
|
|
|
|
# Rate limiting
|
|
WIKIDATA_API_TOKEN = os.getenv("WIKIDATA_API_TOKEN", "")
|
|
WIKIMEDIA_CONTACT_EMAIL = os.getenv("WIKIMEDIA_CONTACT_EMAIL", "glam-data@example.com")
|
|
USER_AGENT = f"GLAMDataExtractor/1.2 ({WIKIMEDIA_CONTACT_EMAIL}) Python/httpx"
|
|
|
|
if WIKIDATA_API_TOKEN:
|
|
REQUEST_DELAY = 0.75 # ~4800 requests per hour
|
|
logger.info("Using authenticated mode: 5,000 req/hr limit")
|
|
else:
|
|
REQUEST_DELAY = 7.5 # ~480 requests per hour
|
|
logger.info("Using anonymous mode: 500 req/hr limit")
|
|
|
|
# HTTP Headers
|
|
HEADERS = {
|
|
"Accept": "application/json",
|
|
"User-Agent": USER_AGENT,
|
|
}
|
|
if WIKIDATA_API_TOKEN:
|
|
HEADERS["Authorization"] = f"Bearer {WIKIDATA_API_TOKEN}"
|
|
|
|
# COMPREHENSIVE Property mapping - capturing ALL useful properties for heritage institutions
|
|
# Organized by category for clarity
|
|
PROPERTY_MAPPING = {
|
|
# === TEMPORAL PROPERTIES ===
|
|
"P571": {"name": "inception", "type": "time", "category": "temporal"},
|
|
"P576": {"name": "dissolution", "type": "time", "category": "temporal"},
|
|
"P1619": {"name": "date_of_official_opening", "type": "time", "category": "temporal"},
|
|
"P580": {"name": "start_time", "type": "time", "category": "temporal"},
|
|
"P582": {"name": "end_time", "type": "time", "category": "temporal"},
|
|
|
|
# === ORGANIZATIONAL PROPERTIES ===
|
|
"P31": {"name": "instance_of", "type": "entity_list", "category": "classification"},
|
|
"P17": {"name": "country", "type": "entity", "category": "location"},
|
|
"P131": {"name": "located_in_admin_entity", "type": "entity", "category": "location"},
|
|
"P276": {"name": "location", "type": "entity", "category": "location"},
|
|
"P159": {"name": "headquarters_location", "type": "entity", "category": "location"},
|
|
"P625": {"name": "coordinates", "type": "coordinates", "category": "location"},
|
|
"P969": {"name": "located_at_street_address", "type": "string", "category": "location"},
|
|
"P281": {"name": "postal_code", "type": "string", "category": "location"},
|
|
"P749": {"name": "parent_organization", "type": "entity", "category": "organization"},
|
|
"P355": {"name": "subsidiary", "type": "entity_list", "category": "organization"},
|
|
"P361": {"name": "part_of", "type": "entity", "category": "organization"},
|
|
"P527": {"name": "has_parts", "type": "entity_list", "category": "organization"},
|
|
"P463": {"name": "member_of", "type": "entity_list", "category": "organization"},
|
|
"P101": {"name": "field_of_work", "type": "entity_list", "category": "classification"},
|
|
"P921": {"name": "main_subject", "type": "entity_list", "category": "classification"},
|
|
"P3032": {"name": "adjacent_building", "type": "entity", "category": "location"},
|
|
"P1435": {"name": "heritage_designation", "type": "entity_list", "category": "classification"},
|
|
"P112": {"name": "founded_by", "type": "entity_list", "category": "organization"},
|
|
"P169": {"name": "chief_executive_officer", "type": "entity", "category": "organization"},
|
|
"P488": {"name": "chairperson", "type": "entity", "category": "organization"},
|
|
|
|
# === IDENTIFIERS ===
|
|
"P791": {"name": "isil", "type": "string", "category": "identifier"},
|
|
"P214": {"name": "viaf", "type": "string", "category": "identifier"},
|
|
"P227": {"name": "gnd", "type": "string", "category": "identifier"},
|
|
"P244": {"name": "lcnaf", "type": "string", "category": "identifier"},
|
|
"P268": {"name": "bnf", "type": "string", "category": "identifier"},
|
|
"P269": {"name": "idref", "type": "string", "category": "identifier"},
|
|
"P213": {"name": "isni", "type": "string", "category": "identifier"},
|
|
"P1566": {"name": "geonames_id", "type": "string", "category": "identifier"},
|
|
"P349": {"name": "ndl_authority_id", "type": "string", "category": "identifier"},
|
|
"P271": {"name": "nacsis_cat_id", "type": "string", "category": "identifier"},
|
|
"P2671": {"name": "google_knowledge_graph_id", "type": "string", "category": "identifier"},
|
|
"P3134": {"name": "tripadvisor_id", "type": "string", "category": "identifier"},
|
|
"P11693": {"name": "openstreetmap_node_id", "type": "string", "category": "identifier"},
|
|
"P11496": {"name": "cinii_research_id", "type": "string", "category": "identifier"},
|
|
"P5587": {"name": "libris_uri", "type": "string", "category": "identifier"},
|
|
"P496": {"name": "orcid", "type": "string", "category": "identifier"},
|
|
"P1015": {"name": "noraf_id", "type": "string", "category": "identifier"},
|
|
"P1006": {"name": "nta_id", "type": "string", "category": "identifier"},
|
|
"P409": {"name": "nla_id", "type": "string", "category": "identifier"},
|
|
"P950": {"name": "bne_id", "type": "string", "category": "identifier"},
|
|
"P906": {"name": "selibr", "type": "string", "category": "identifier"},
|
|
"P1017": {"name": "bac_id", "type": "string", "category": "identifier"},
|
|
"P7859": {"name": "worldcat_identities_id", "type": "string", "category": "identifier"},
|
|
"P3500": {"name": "ringgold_id", "type": "string", "category": "identifier"},
|
|
"P2427": {"name": "grid_id", "type": "string", "category": "identifier"},
|
|
"P6782": {"name": "ror_id", "type": "string", "category": "identifier"},
|
|
"P3153": {"name": "crossref_funder_id", "type": "string", "category": "identifier"},
|
|
|
|
# === WEB PRESENCE ===
|
|
"P856": {"name": "official_website", "type": "url", "category": "web"},
|
|
"P1581": {"name": "official_blog_url", "type": "url", "category": "web"},
|
|
"P973": {"name": "described_at_url", "type": "url", "category": "web"},
|
|
"P2013": {"name": "facebook_id", "type": "string", "category": "social"},
|
|
"P2002": {"name": "twitter_username", "type": "string", "category": "social"},
|
|
"P2003": {"name": "instagram_username", "type": "string", "category": "social"},
|
|
"P2397": {"name": "youtube_channel_id", "type": "string", "category": "social"},
|
|
"P4264": {"name": "linkedin_company_id", "type": "string", "category": "social"},
|
|
"P4003": {"name": "facebook_page_id", "type": "string", "category": "social"},
|
|
"P8687": {"name": "social_media_followers", "type": "quantity", "category": "social"},
|
|
|
|
# === MEDIA ===
|
|
"P18": {"name": "image", "type": "commons_media", "category": "media"},
|
|
"P154": {"name": "logo", "type": "commons_media", "category": "media"},
|
|
"P41": {"name": "flag_image", "type": "commons_media", "category": "media"},
|
|
"P94": {"name": "coat_of_arms", "type": "commons_media", "category": "media"},
|
|
"P373": {"name": "commons_category", "type": "string", "category": "media"},
|
|
"P935": {"name": "commons_gallery", "type": "string", "category": "media"},
|
|
|
|
# === CONTACT ===
|
|
"P968": {"name": "email", "type": "string", "category": "contact"},
|
|
"P1329": {"name": "phone_number", "type": "string", "category": "contact"},
|
|
"P3740": {"name": "number_of_works", "type": "quantity", "category": "collection"},
|
|
"P1436": {"name": "collection_items_count", "type": "quantity", "category": "collection"},
|
|
|
|
# === AWARDS & RECOGNITION ===
|
|
"P166": {"name": "award_received", "type": "entity_list", "category": "recognition"},
|
|
|
|
# === ARCHITECTURE ===
|
|
"P149": {"name": "architectural_style", "type": "entity_list", "category": "architecture"},
|
|
"P84": {"name": "architect", "type": "entity_list", "category": "architecture"},
|
|
"P631": {"name": "structural_engineer", "type": "entity_list", "category": "architecture"},
|
|
}
|
|
|
|
# CRITICAL KEYS that must NEVER be deleted during enrichment
|
|
# See AGENTS.md Rule 5: NEVER Delete Enriched Data - Additive Only
|
|
PROTECTED_KEYS = {
|
|
'location', 'original_entry', 'ghcid', 'custodian_name', 'identifiers',
|
|
'provenance', 'ch_annotator', 'google_maps_enrichment', 'osm_enrichment',
|
|
'unesco_mow_enrichment', 'web_enrichment', 'linkedin_enrichment',
|
|
'zcbs_enrichment', 'person_observations'
|
|
}
|
|
|
|
|
|
@dataclass
|
|
class FullWikidataEnrichment:
|
|
"""Container for comprehensive Wikidata enrichment data."""
|
|
entity_id: str
|
|
labels: Dict[str, str] = field(default_factory=dict)
|
|
descriptions: Dict[str, str] = field(default_factory=dict)
|
|
aliases: Dict[str, List[str]] = field(default_factory=dict)
|
|
sitelinks: Dict[str, str] = field(default_factory=dict)
|
|
|
|
# All extracted properties organized by category
|
|
temporal: Dict[str, Any] = field(default_factory=dict)
|
|
classification: Dict[str, Any] = field(default_factory=dict)
|
|
location: Dict[str, Any] = field(default_factory=dict)
|
|
organization: Dict[str, Any] = field(default_factory=dict)
|
|
identifiers: Dict[str, str] = field(default_factory=dict)
|
|
web: Dict[str, str] = field(default_factory=dict)
|
|
social: Dict[str, str] = field(default_factory=dict)
|
|
media: Dict[str, str] = field(default_factory=dict)
|
|
contact: Dict[str, str] = field(default_factory=dict)
|
|
collection: Dict[str, Any] = field(default_factory=dict)
|
|
recognition: Dict[str, Any] = field(default_factory=dict)
|
|
architecture: Dict[str, Any] = field(default_factory=dict)
|
|
|
|
# Metadata
|
|
fetch_timestamp: str = field(default_factory=lambda: datetime.now(timezone.utc).isoformat())
|
|
properties_found: List[str] = field(default_factory=list)
|
|
|
|
|
|
def extract_value_from_statement(statement: Dict) -> Any:
|
|
"""Extract the value from a Wikidata statement structure."""
|
|
try:
|
|
value_data = statement.get("value", {})
|
|
content = value_data.get("content")
|
|
|
|
if isinstance(content, dict):
|
|
if "entity-type" in content or "id" in content:
|
|
return content.get("id", content)
|
|
elif "time" in content:
|
|
time_val = content.get("time", "")
|
|
if time_val.startswith("+") or time_val.startswith("-"):
|
|
time_val = time_val[1:]
|
|
if "T" in time_val:
|
|
time_val = time_val.split("T")[0]
|
|
return time_val
|
|
elif "latitude" in content and "longitude" in content:
|
|
return {
|
|
"latitude": content.get("latitude"),
|
|
"longitude": content.get("longitude"),
|
|
"precision": content.get("precision")
|
|
}
|
|
elif "amount" in content:
|
|
return content.get("amount", "").lstrip("+")
|
|
else:
|
|
return content
|
|
else:
|
|
return content
|
|
except Exception:
|
|
return None
|
|
|
|
|
|
def fetch_entity_labels_batch(entity_ids: Set[str], client: httpx.Client) -> Dict[str, Dict[str, str]]:
|
|
"""Fetch labels for multiple entities in a batch using SPARQL."""
|
|
if not entity_ids:
|
|
return {}
|
|
|
|
# Limit batch size
|
|
entity_ids_list = list(entity_ids)[:50]
|
|
|
|
entity_values = " ".join([f"wd:{eid}" for eid in entity_ids_list])
|
|
query = f"""
|
|
SELECT ?entity ?entityLabel ?entityDescription WHERE {{
|
|
VALUES ?entity {{ {entity_values} }}
|
|
SERVICE wikibase:label {{ bd:serviceParam wikibase:language "en,ja,nl,de,fr,cs,zh". }}
|
|
}}
|
|
"""
|
|
|
|
try:
|
|
response = client.get(
|
|
WIKIDATA_SPARQL_ENDPOINT,
|
|
params={"query": query, "format": "json"},
|
|
headers={"User-Agent": USER_AGENT, "Accept": "application/sparql-results+json"}
|
|
)
|
|
response.raise_for_status()
|
|
results = response.json()
|
|
|
|
labels = {}
|
|
for binding in results.get("results", {}).get("bindings", []):
|
|
entity_uri = binding.get("entity", {}).get("value", "")
|
|
entity_id = entity_uri.split("/")[-1] if entity_uri else None
|
|
if entity_id:
|
|
labels[entity_id] = {
|
|
"id": entity_id,
|
|
"label": binding.get("entityLabel", {}).get("value", entity_id),
|
|
"description": binding.get("entityDescription", {}).get("value", "")
|
|
}
|
|
return labels
|
|
except Exception as e:
|
|
logger.warning(f"SPARQL label fetch failed: {e}")
|
|
return {eid: {"id": eid, "label": eid} for eid in entity_ids_list}
|
|
|
|
|
|
def fetch_entity_data(entity_id: str, client: httpx.Client) -> Optional[Dict]:
|
|
"""Fetch full entity data from Wikibase REST API."""
|
|
url = f"{WIKIDATA_REST_API}/entities/items/{entity_id}"
|
|
|
|
try:
|
|
response = client.get(url, headers=HEADERS)
|
|
|
|
if response.status_code == 403:
|
|
headers_no_auth = {k: v for k, v in HEADERS.items() if k != "Authorization"}
|
|
response = client.get(url, headers=headers_no_auth)
|
|
|
|
response.raise_for_status()
|
|
return response.json()
|
|
|
|
except httpx.HTTPStatusError as e:
|
|
if e.response.status_code == 404:
|
|
logger.warning(f"Entity {entity_id} not found")
|
|
else:
|
|
logger.error(f"HTTP error fetching {entity_id}: {e}")
|
|
return None
|
|
except Exception as e:
|
|
logger.error(f"Error fetching {entity_id}: {e}")
|
|
return None
|
|
|
|
|
|
def resolve_isil_to_wikidata(isil_code: str, client: httpx.Client) -> Optional[str]:
|
|
"""Resolve an ISIL code to a Wikidata entity ID via P791."""
|
|
query = f"""
|
|
SELECT ?item WHERE {{
|
|
?item wdt:P791 "{isil_code}" .
|
|
}}
|
|
LIMIT 1
|
|
"""
|
|
|
|
try:
|
|
response = client.get(
|
|
WIKIDATA_SPARQL_ENDPOINT,
|
|
params={"query": query, "format": "json"},
|
|
headers={"User-Agent": USER_AGENT, "Accept": "application/sparql-results+json"}
|
|
)
|
|
response.raise_for_status()
|
|
results = response.json()
|
|
|
|
bindings = results.get("results", {}).get("bindings", [])
|
|
if bindings:
|
|
entity_uri = bindings[0].get("item", {}).get("value", "")
|
|
entity_id = entity_uri.split("/")[-1] if entity_uri else None
|
|
if entity_id and entity_id.startswith("Q"):
|
|
logger.info(f" Resolved ISIL {isil_code} → {entity_id}")
|
|
return entity_id
|
|
return None
|
|
except Exception as e:
|
|
logger.warning(f"SPARQL ISIL resolution failed for {isil_code}: {e}")
|
|
return None
|
|
|
|
|
|
def parse_entity_data_full(entity_id: str, data: Dict, client: httpx.Client) -> FullWikidataEnrichment:
|
|
"""Parse the full entity data with label resolution."""
|
|
enrichment = FullWikidataEnrichment(entity_id=entity_id)
|
|
|
|
# Extract labels
|
|
enrichment.labels = data.get("labels", {})
|
|
enrichment.descriptions = data.get("descriptions", {})
|
|
enrichment.aliases = data.get("aliases", {})
|
|
|
|
# Extract sitelinks
|
|
sitelinks = data.get("sitelinks", {})
|
|
enrichment.sitelinks = {k: v.get("title", "") for k, v in sitelinks.items() if isinstance(v, dict)}
|
|
|
|
# Collect entity IDs that need label resolution
|
|
entity_ids_to_resolve: Set[str] = set()
|
|
|
|
# Process all statements
|
|
statements = data.get("statements", {})
|
|
|
|
for prop_id, prop_statements in statements.items():
|
|
if not prop_statements:
|
|
continue
|
|
|
|
prop_config = PROPERTY_MAPPING.get(prop_id)
|
|
if not prop_config:
|
|
continue # Skip unknown properties
|
|
|
|
enrichment.properties_found.append(prop_id)
|
|
prop_name: str = prop_config["name"]
|
|
prop_type: str = prop_config["type"]
|
|
category: str = prop_config["category"]
|
|
|
|
values: List[Any] = []
|
|
for stmt in prop_statements:
|
|
value = extract_value_from_statement(stmt)
|
|
if value is not None:
|
|
values.append(value)
|
|
# Collect entity IDs for label resolution
|
|
if prop_type in ("entity", "entity_list") and isinstance(value, str) and value.startswith("Q"):
|
|
entity_ids_to_resolve.add(value)
|
|
|
|
if not values:
|
|
continue
|
|
|
|
# Store values in appropriate category
|
|
target_dict = getattr(enrichment, category, None)
|
|
if target_dict is None:
|
|
continue
|
|
|
|
if prop_type == "entity":
|
|
target_dict[prop_name] = values[0]
|
|
elif prop_type == "entity_list":
|
|
target_dict[prop_name] = values
|
|
elif prop_type in ("string", "url"):
|
|
target_dict[prop_name] = values[0] if len(values) == 1 else values
|
|
elif prop_type == "time":
|
|
target_dict[prop_name] = values[0]
|
|
elif prop_type == "coordinates":
|
|
target_dict[prop_name] = values[0]
|
|
elif prop_type == "commons_media":
|
|
target_dict[prop_name] = values[0]
|
|
elif prop_type == "quantity":
|
|
target_dict[prop_name] = values[0]
|
|
|
|
# Resolve entity labels
|
|
if entity_ids_to_resolve:
|
|
time.sleep(0.2) # Small delay before SPARQL query
|
|
labels_map = fetch_entity_labels_batch(entity_ids_to_resolve, client)
|
|
|
|
# Replace entity IDs with resolved labels
|
|
for category_name in ["classification", "location", "organization", "recognition", "architecture"]:
|
|
category_dict = getattr(enrichment, category_name, {})
|
|
for key, value in list(category_dict.items()):
|
|
if isinstance(value, str) and value in labels_map:
|
|
category_dict[key] = labels_map[value]
|
|
elif isinstance(value, list):
|
|
category_dict[key] = [
|
|
labels_map.get(v, {"id": v, "label": v}) if isinstance(v, str) and v.startswith("Q") else v
|
|
for v in value
|
|
]
|
|
|
|
return enrichment
|
|
|
|
|
|
def enrichment_to_dict(enrichment: FullWikidataEnrichment) -> Dict:
|
|
"""Convert FullWikidataEnrichment to a dictionary for YAML output."""
|
|
result = {
|
|
"wikidata_entity_id": enrichment.entity_id,
|
|
"api_metadata": {
|
|
"api_endpoint": WIKIDATA_REST_API,
|
|
"fetch_timestamp": enrichment.fetch_timestamp,
|
|
"user_agent": USER_AGENT,
|
|
"enrichment_version": "2.1_generic",
|
|
"properties_found": enrichment.properties_found,
|
|
}
|
|
}
|
|
|
|
# Add labels
|
|
if enrichment.labels:
|
|
result["wikidata_labels"] = enrichment.labels
|
|
for lang in ["en", "nl", "ja", "de", "fr", "es", "cs", "zh"]:
|
|
if lang in enrichment.labels:
|
|
result[f"wikidata_label_{lang}"] = enrichment.labels[lang]
|
|
|
|
# Add descriptions
|
|
if enrichment.descriptions:
|
|
result["wikidata_descriptions"] = enrichment.descriptions
|
|
if "en" in enrichment.descriptions:
|
|
result["wikidata_description_en"] = enrichment.descriptions["en"]
|
|
|
|
# Add aliases
|
|
if enrichment.aliases:
|
|
result["wikidata_aliases"] = enrichment.aliases
|
|
|
|
# Add sitelinks (Wikipedia articles)
|
|
if enrichment.sitelinks:
|
|
result["wikidata_sitelinks"] = enrichment.sitelinks
|
|
|
|
# Add all category data with readable prefixes
|
|
if enrichment.temporal:
|
|
result["wikidata_temporal"] = enrichment.temporal
|
|
# Promote key dates to top level for easy access
|
|
if "inception" in enrichment.temporal:
|
|
result["wikidata_inception"] = enrichment.temporal["inception"]
|
|
if "dissolution" in enrichment.temporal:
|
|
result["wikidata_dissolution"] = enrichment.temporal["dissolution"]
|
|
if "date_of_official_opening" in enrichment.temporal:
|
|
result["wikidata_opening_date"] = enrichment.temporal["date_of_official_opening"]
|
|
|
|
if enrichment.classification:
|
|
result["wikidata_classification"] = enrichment.classification
|
|
if "instance_of" in enrichment.classification:
|
|
result["wikidata_instance_of"] = enrichment.classification["instance_of"]
|
|
if "field_of_work" in enrichment.classification:
|
|
result["wikidata_field_of_work"] = enrichment.classification["field_of_work"]
|
|
|
|
if enrichment.location:
|
|
result["wikidata_location"] = enrichment.location
|
|
if "country" in enrichment.location:
|
|
result["wikidata_country"] = enrichment.location["country"]
|
|
if "located_in_admin_entity" in enrichment.location:
|
|
result["wikidata_located_in"] = enrichment.location["located_in_admin_entity"]
|
|
if "coordinates" in enrichment.location:
|
|
result["wikidata_coordinates"] = enrichment.location["coordinates"]
|
|
|
|
if enrichment.organization:
|
|
result["wikidata_organization"] = enrichment.organization
|
|
|
|
if enrichment.identifiers:
|
|
result["wikidata_identifiers"] = enrichment.identifiers
|
|
|
|
if enrichment.web:
|
|
result["wikidata_web"] = enrichment.web
|
|
if "official_website" in enrichment.web:
|
|
result["wikidata_official_website"] = enrichment.web["official_website"]
|
|
|
|
if enrichment.social:
|
|
result["wikidata_social_media"] = enrichment.social
|
|
|
|
if enrichment.media:
|
|
result["wikidata_media"] = enrichment.media
|
|
if "image" in enrichment.media:
|
|
result["wikidata_image"] = enrichment.media["image"]
|
|
if "logo" in enrichment.media:
|
|
result["wikidata_logo"] = enrichment.media["logo"]
|
|
|
|
if enrichment.contact:
|
|
result["wikidata_contact"] = enrichment.contact
|
|
|
|
if enrichment.collection:
|
|
result["wikidata_collection"] = enrichment.collection
|
|
|
|
if enrichment.recognition:
|
|
result["wikidata_recognition"] = enrichment.recognition
|
|
|
|
if enrichment.architecture:
|
|
result["wikidata_architecture"] = enrichment.architecture
|
|
|
|
return result
|
|
|
|
|
|
def get_wikidata_entity_id(data: Dict) -> Optional[str]:
|
|
"""
|
|
Extract Wikidata entity ID from a custodian YAML file.
|
|
|
|
Checks multiple locations where Wikidata ID might be stored:
|
|
1. wikidata_enrichment.wikidata_entity_id
|
|
2. identifiers[].identifier_scheme == "Wikidata"
|
|
3. original_entry.wikidata_id
|
|
4. original_entry.identifiers[].identifier_scheme == "Wikidata"
|
|
"""
|
|
# Check wikidata_enrichment
|
|
wd = data.get("wikidata_enrichment", {})
|
|
if wd and wd.get("wikidata_entity_id"):
|
|
return wd.get("wikidata_entity_id")
|
|
|
|
# Check identifiers list
|
|
identifiers = data.get("identifiers", [])
|
|
for ident in identifiers:
|
|
if isinstance(ident, dict):
|
|
scheme = ident.get("identifier_scheme", "")
|
|
if scheme.lower() == "wikidata":
|
|
return ident.get("identifier_value")
|
|
|
|
# Check original_entry
|
|
original = data.get("original_entry", {})
|
|
|
|
# Direct wikidata_id in original_entry
|
|
if original.get("wikidata_id"):
|
|
return original.get("wikidata_id")
|
|
|
|
# Identifiers in original_entry
|
|
for ident in original.get("identifiers", []):
|
|
if isinstance(ident, dict):
|
|
scheme = ident.get("identifier_scheme", "")
|
|
if scheme.lower() == "wikidata":
|
|
return ident.get("identifier_value")
|
|
|
|
return None
|
|
|
|
|
|
def get_isil_code(data: Dict) -> Optional[str]:
|
|
"""
|
|
Extract ISIL code from a custodian YAML file.
|
|
|
|
Checks multiple locations where ISIL might be stored:
|
|
1. identifiers[].identifier_scheme == "ISIL"
|
|
2. original_entry.isil_code
|
|
3. original_entry.identifiers[].identifier_scheme == "ISIL"
|
|
4. wikidata_enrichment.wikidata_identifiers.isil
|
|
"""
|
|
# Check identifiers list
|
|
identifiers = data.get("identifiers", [])
|
|
for ident in identifiers:
|
|
if isinstance(ident, dict):
|
|
scheme = ident.get("identifier_scheme", "")
|
|
if scheme.lower() == "isil":
|
|
return ident.get("identifier_value")
|
|
|
|
# Check original_entry
|
|
original = data.get("original_entry", {})
|
|
|
|
if original.get("isil_code"):
|
|
return original.get("isil_code")
|
|
|
|
for ident in original.get("identifiers", []):
|
|
if isinstance(ident, dict):
|
|
scheme = ident.get("identifier_scheme", "")
|
|
if scheme.lower() == "isil":
|
|
return ident.get("identifier_value")
|
|
|
|
# Check wikidata_enrichment.wikidata_identifiers
|
|
wd = data.get("wikidata_enrichment", {})
|
|
if wd.get("wikidata_identifiers", {}).get("isil"):
|
|
return wd.get("wikidata_identifiers", {}).get("isil")
|
|
|
|
return None
|
|
|
|
|
|
def is_fully_enriched(data: Dict) -> bool:
|
|
"""Check if file has been fully enriched with v2.0+."""
|
|
wd = data.get("wikidata_enrichment", {})
|
|
api_meta = wd.get("api_metadata", {})
|
|
version = api_meta.get("enrichment_version", "")
|
|
return version.startswith("2.") if version else False
|
|
|
|
|
|
def load_progress(progress_file: Path) -> Dict:
|
|
"""Load progress from checkpoint file."""
|
|
if progress_file.exists():
|
|
try:
|
|
with open(progress_file, 'r') as f:
|
|
return json.load(f)
|
|
except Exception:
|
|
pass
|
|
return {"processed_files": [], "stats": {}, "isil_resolutions": {}}
|
|
|
|
|
|
def save_progress(progress: Dict, progress_file: Path):
|
|
"""Save progress to checkpoint file."""
|
|
try:
|
|
with open(progress_file, 'w') as f:
|
|
json.dump(progress, f, indent=2)
|
|
except Exception as e:
|
|
logger.error(f"Failed to save progress: {e}")
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(
|
|
description="Generic Wikidata enrichment for custodian files",
|
|
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
epilog="""
|
|
Examples:
|
|
# Dry run on Japan files (first 10)
|
|
python scripts/enrich_custodians_generic.py --country JP --limit 10 --dry-run
|
|
|
|
# Enrich all Czech files
|
|
python scripts/enrich_custodians_generic.py --country CZ
|
|
|
|
# Force re-enrichment of already enriched files
|
|
python scripts/enrich_custodians_generic.py --country JP --force
|
|
|
|
# Resume from last checkpoint
|
|
python scripts/enrich_custodians_generic.py --country JP --resume
|
|
"""
|
|
)
|
|
parser.add_argument("--country", type=str, help="Only process files for country code XX (e.g., JP, CZ, NL)")
|
|
parser.add_argument("--dry-run", action="store_true", help="Show what would be enriched without modifying files")
|
|
parser.add_argument("--limit", type=int, default=0, help="Process only first N files (0 = no limit)")
|
|
parser.add_argument("--force", action="store_true", help="Re-enrich even if already has v2.0+ enrichment")
|
|
parser.add_argument("--resume", action="store_true", help="Resume from last checkpoint")
|
|
parser.add_argument("--verbose", action="store_true", help="Show detailed progress information")
|
|
parser.add_argument("--resolve-isil", action="store_true", help="Try to resolve ISIL codes to Wikidata IDs")
|
|
args = parser.parse_args()
|
|
|
|
if args.verbose:
|
|
logger.setLevel(logging.DEBUG)
|
|
|
|
# Use country-specific progress file if country is specified
|
|
progress_file = PROGRESS_FILE
|
|
if args.country:
|
|
progress_file = CUSTODIAN_DIR / f".enrichment_progress_{args.country}.json"
|
|
|
|
progress = load_progress(progress_file) if args.resume else {"processed_files": [], "stats": {}, "isil_resolutions": {}}
|
|
processed_files = set(progress.get("processed_files", []))
|
|
isil_resolutions = progress.get("isil_resolutions", {}) # Cache ISIL → Wikidata mappings
|
|
|
|
stats = {
|
|
"total_scanned": 0,
|
|
"needs_enrichment": 0,
|
|
"already_enriched_v2": 0,
|
|
"no_wikidata_id": 0,
|
|
"has_isil_only": 0,
|
|
"isil_resolved": 0,
|
|
"enriched_successfully": 0,
|
|
"errors": 0,
|
|
"skipped_already_processed": 0,
|
|
"properties_counts": {},
|
|
}
|
|
|
|
pattern = f"{args.country}-*.yaml" if args.country else "*.yaml"
|
|
yaml_files = sorted(CUSTODIAN_DIR.glob(pattern))
|
|
|
|
logger.info(f"Found {len(yaml_files)} YAML files matching pattern '{pattern}'")
|
|
|
|
files_to_process = []
|
|
|
|
# First pass: scan files and identify candidates
|
|
logger.info("Scanning files to identify enrichment candidates...")
|
|
for yaml_file in yaml_files:
|
|
stats["total_scanned"] += 1
|
|
|
|
if args.resume and yaml_file.name in processed_files:
|
|
stats["skipped_already_processed"] += 1
|
|
continue
|
|
|
|
try:
|
|
with open(yaml_file, 'r', encoding='utf-8') as f:
|
|
data = yaml.safe_load(f)
|
|
|
|
if not data:
|
|
continue
|
|
|
|
entity_id = get_wikidata_entity_id(data)
|
|
|
|
if not entity_id and args.resolve_isil:
|
|
# Try to find ISIL code
|
|
isil_code = get_isil_code(data)
|
|
if isil_code:
|
|
stats["has_isil_only"] += 1
|
|
# Check cache first
|
|
if isil_code in isil_resolutions:
|
|
entity_id = isil_resolutions[isil_code]
|
|
if entity_id:
|
|
stats["isil_resolved"] += 1
|
|
else:
|
|
# Will resolve later during processing
|
|
pass
|
|
|
|
if not entity_id:
|
|
stats["no_wikidata_id"] += 1
|
|
# Still add to process list if we might resolve ISIL
|
|
if args.resolve_isil:
|
|
isil_code = get_isil_code(data)
|
|
if isil_code and isil_code not in isil_resolutions:
|
|
files_to_process.append((yaml_file, None, isil_code))
|
|
continue
|
|
|
|
if not args.force and is_fully_enriched(data):
|
|
stats["already_enriched_v2"] += 1
|
|
continue
|
|
|
|
stats["needs_enrichment"] += 1
|
|
files_to_process.append((yaml_file, entity_id, None))
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error reading {yaml_file}: {e}")
|
|
stats["errors"] += 1
|
|
|
|
logger.info(f"\n{'='*60}")
|
|
logger.info(f"SCAN COMPLETE")
|
|
logger.info(f"{'='*60}")
|
|
logger.info(f"Total files scanned: {stats['total_scanned']}")
|
|
logger.info(f"Files needing enrichment: {stats['needs_enrichment']}")
|
|
logger.info(f"Files already enriched (v2.0+): {stats['already_enriched_v2']}")
|
|
logger.info(f"Files without Wikidata ID: {stats['no_wikidata_id']}")
|
|
if args.resolve_isil:
|
|
logger.info(f"Files with ISIL only: {stats['has_isil_only']}")
|
|
logger.info(f"Skipped (already processed): {stats['skipped_already_processed']}")
|
|
logger.info(f"{'='*60}\n")
|
|
|
|
if args.limit > 0:
|
|
files_to_process = files_to_process[:args.limit]
|
|
logger.info(f"Limited to first {args.limit} files")
|
|
|
|
if args.dry_run:
|
|
logger.info("DRY RUN - No files will be modified")
|
|
logger.info(f"\nFiles to process ({len(files_to_process)}):")
|
|
for yaml_file, entity_id, isil_code in files_to_process[:20]:
|
|
if entity_id:
|
|
logger.info(f" {yaml_file.name} → Wikidata: {entity_id}")
|
|
elif isil_code:
|
|
logger.info(f" {yaml_file.name} → ISIL: {isil_code} (needs resolution)")
|
|
if len(files_to_process) > 20:
|
|
logger.info(f" ... and {len(files_to_process) - 20} more")
|
|
return
|
|
|
|
# Second pass: process files
|
|
with httpx.Client(timeout=30.0) as client:
|
|
for i, (yaml_file, entity_id, isil_code) in enumerate(files_to_process):
|
|
try:
|
|
# Resolve ISIL to Wikidata if needed
|
|
if not entity_id and isil_code:
|
|
if isil_code in isil_resolutions:
|
|
entity_id = isil_resolutions[isil_code]
|
|
else:
|
|
logger.info(f"[{i+1}/{len(files_to_process)}] Resolving ISIL {isil_code}...")
|
|
entity_id = resolve_isil_to_wikidata(isil_code, client)
|
|
isil_resolutions[isil_code] = entity_id
|
|
time.sleep(REQUEST_DELAY)
|
|
|
|
if entity_id:
|
|
stats["isil_resolved"] += 1
|
|
else:
|
|
logger.info(f" Could not resolve ISIL {isil_code}")
|
|
continue
|
|
|
|
if not entity_id:
|
|
continue
|
|
|
|
logger.info(f"[{i+1}/{len(files_to_process)}] Enriching {yaml_file.name} ({entity_id})")
|
|
|
|
# Re-read the file immediately before modifying
|
|
with open(yaml_file, 'r', encoding='utf-8') as f:
|
|
data = yaml.safe_load(f)
|
|
|
|
if not data:
|
|
logger.warning(f" File is empty or invalid: {yaml_file.name}")
|
|
stats["errors"] += 1
|
|
continue
|
|
|
|
# Record which protected keys exist BEFORE modification
|
|
keys_before = set(data.keys())
|
|
protected_keys_before = keys_before & PROTECTED_KEYS
|
|
|
|
entity_data = fetch_entity_data(entity_id, client)
|
|
|
|
if entity_data is None:
|
|
logger.warning(f" Could not fetch data for {entity_id}")
|
|
stats["errors"] += 1
|
|
continue
|
|
|
|
enrichment = parse_entity_data_full(entity_id, entity_data, client)
|
|
enrichment_dict = enrichment_to_dict(enrichment)
|
|
|
|
data["wikidata_enrichment"] = enrichment_dict
|
|
|
|
# SAFETY CHECK: Verify no protected keys were lost
|
|
keys_after = set(data.keys())
|
|
protected_keys_after = keys_after & PROTECTED_KEYS
|
|
lost_keys = protected_keys_before - protected_keys_after
|
|
if lost_keys:
|
|
logger.error(f" CRITICAL: Protected keys lost during enrichment: {lost_keys}")
|
|
logger.error(f" Skipping file to prevent data loss!")
|
|
stats["errors"] += 1
|
|
continue
|
|
|
|
# Track property statistics
|
|
for prop in enrichment.properties_found:
|
|
stats["properties_counts"][prop] = stats["properties_counts"].get(prop, 0) + 1
|
|
|
|
stats["enriched_successfully"] += 1
|
|
|
|
# Log key findings
|
|
findings = []
|
|
if enrichment.temporal.get("inception"):
|
|
findings.append(f"inception: {enrichment.temporal['inception']}")
|
|
if enrichment.temporal.get("date_of_official_opening"):
|
|
findings.append(f"opened: {enrichment.temporal['date_of_official_opening']}")
|
|
if enrichment.classification.get("field_of_work"):
|
|
fow = enrichment.classification["field_of_work"]
|
|
if isinstance(fow, list) and fow:
|
|
label = fow[0].get("label", fow[0]) if isinstance(fow[0], dict) else fow[0]
|
|
findings.append(f"field: {label}")
|
|
if enrichment.identifiers:
|
|
findings.append(f"{len(enrichment.identifiers)} identifiers")
|
|
|
|
if findings:
|
|
logger.info(f" Found: {', '.join(findings)}")
|
|
|
|
with open(yaml_file, 'w', encoding='utf-8') as f:
|
|
yaml.dump(data, f, default_flow_style=False, allow_unicode=True, sort_keys=False)
|
|
|
|
processed_files.add(yaml_file.name)
|
|
progress["processed_files"] = list(processed_files)
|
|
progress["stats"] = stats
|
|
progress["isil_resolutions"] = isil_resolutions
|
|
|
|
if (i + 1) % 10 == 0:
|
|
save_progress(progress, progress_file)
|
|
|
|
time.sleep(REQUEST_DELAY)
|
|
|
|
except Exception as e:
|
|
logger.error(f"Error processing {yaml_file.name}: {e}")
|
|
stats["errors"] += 1
|
|
|
|
save_progress(progress, progress_file)
|
|
|
|
logger.info("\n" + "=" * 60)
|
|
logger.info("ENRICHMENT COMPLETE")
|
|
logger.info("=" * 60)
|
|
logger.info(f"Total files scanned: {stats['total_scanned']}")
|
|
logger.info(f"Files needing enrichment: {stats['needs_enrichment']}")
|
|
logger.info(f"Already enriched (v2.0+): {stats['already_enriched_v2']}")
|
|
logger.info(f"Files without Wikidata ID: {stats['no_wikidata_id']}")
|
|
if args.resolve_isil:
|
|
logger.info(f"ISIL codes resolved: {stats['isil_resolved']}")
|
|
logger.info(f"Successfully enriched: {stats['enriched_successfully']}")
|
|
logger.info(f"Errors: {stats['errors']}")
|
|
logger.info("")
|
|
logger.info("Top properties found:")
|
|
sorted_props = sorted(stats["properties_counts"].items(), key=lambda x: x[1], reverse=True)[:15]
|
|
for prop, count in sorted_props:
|
|
prop_name = PROPERTY_MAPPING.get(prop, {}).get("name", prop)
|
|
logger.info(f" {prop} ({prop_name}): {count}")
|
|
logger.info("=" * 60)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|