e5a532a8bc
- Introduced `test_nlp_extractor.py` with unit tests for the InstitutionExtractor, covering various extraction patterns (ISIL, Wikidata, VIAF, city names) and ensuring proper classification of institutions (museum, library, archive). - Added tests for extracted entities and result handling to validate the extraction process. - Created `test_partnership_rdf_integration.py` to validate the end-to-end process of extracting partnerships from a conversation and exporting them to RDF format. - Implemented tests for temporal properties in partnerships and ensured compliance with W3C Organization Ontology patterns. - Verified that extracted partnerships are correctly linked with PROV-O provenance metadata.
206 lines
8.7 KiB
Python
206 lines
8.7 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Belgium Heritage Institutions Enrichment - Manual Matches
|
|
=========================================================
|
|
|
|
Strategy: All 7 Belgian institutions are EU institutions in Brussels.
|
|
Link archives and libraries to their parent EU organizations.
|
|
|
|
Manual Research Findings:
|
|
1. European Committee of the Regions → Q202479 (direct)
|
|
2. European Parliament - Library → Q8889 (parent org)
|
|
3. General Secretariat of the Council - Archives → Q8896 (parent org: Council of EU)
|
|
4. General Secretariat of the Council - Library → Q8896 (parent org: Council of EU)
|
|
5. European Commission - Archives → Q8880 (parent org)
|
|
6. European Commission - Library → Q8880 (parent org)
|
|
7. European Economic and Social Committee → Q641817 (direct)
|
|
|
|
Target: 7 BE institutions → 100% coverage
|
|
"""
|
|
|
|
import yaml
|
|
from datetime import datetime, timezone
|
|
import os
|
|
|
|
def apply_manual_matches():
|
|
"""Apply manually researched Wikidata matches for EU institutions."""
|
|
|
|
print("=" * 80)
|
|
print("🇧🇪 Belgium Heritage Institutions Enrichment - Manual Matches")
|
|
print("=" * 80)
|
|
print("\nStrategy: EU institutions in Brussels - link to parent organizations\n")
|
|
|
|
# Load unified dataset
|
|
print("📂 Loading unified global dataset...")
|
|
with open('data/instances/all/globalglam-20251111.yaml', 'r', encoding='utf-8') as f:
|
|
all_institutions = yaml.safe_load(f)
|
|
|
|
# Filter Belgian institutions
|
|
be_institutions = [
|
|
inst for inst in all_institutions
|
|
if any(loc.get('country') == 'BE' for loc in inst.get('locations', []))
|
|
]
|
|
print(f" ✅ Found {len(be_institutions)} Belgian institutions\n")
|
|
|
|
# Manual match mappings
|
|
manual_matches = {
|
|
'European Committee of the Regions': {
|
|
'q_number': 'Q202479',
|
|
'label': 'European Committee of the Regions',
|
|
'relation': 'EU advisory body',
|
|
'viaf': '148985051',
|
|
'coordinates': (50.8467, 4.3772), # Brussels EU quarter
|
|
'notes': 'EU advisory body established 1994, represents local and regional authorities'
|
|
},
|
|
'European Parliament - Library': {
|
|
'q_number': 'Q8889',
|
|
'label': 'European Parliament',
|
|
'relation': 'Library of',
|
|
'viaf': '158939804',
|
|
'coordinates': (50.8467, 4.3772),
|
|
'notes': 'European Parliament Library serves MEPs and parliamentary staff'
|
|
},
|
|
'General Secretariat of the Council - Council Archives': {
|
|
'q_number': 'Q8896',
|
|
'label': 'Council of the European Union',
|
|
'relation': 'Archives of',
|
|
'viaf': '123526698',
|
|
'coordinates': (50.8467, 4.3772),
|
|
'notes': 'Council Archives managed by General Secretariat, preserves EU Council decisions'
|
|
},
|
|
'General Secretariat of the Council - Council Library': {
|
|
'q_number': 'Q8896',
|
|
'label': 'Council of the European Union',
|
|
'relation': 'Library of',
|
|
'viaf': '123526698',
|
|
'coordinates': (50.8467, 4.3772),
|
|
'notes': 'Council Library managed by General Secretariat, supports Council work'
|
|
},
|
|
'European Commission - Archives': {
|
|
'q_number': 'Q8880',
|
|
'label': 'European Commission',
|
|
'relation': 'Archives of',
|
|
'viaf': '144763055',
|
|
'coordinates': (50.8467, 4.3772),
|
|
'notes': 'European Commission Historical Archives preserve executive documentation'
|
|
},
|
|
'European Commission - European Commission Library': {
|
|
'q_number': 'Q8880',
|
|
'label': 'European Commission',
|
|
'relation': 'Library of',
|
|
'viaf': '144763055',
|
|
'coordinates': (50.8467, 4.3772),
|
|
'notes': 'Central Library of the European Commission in Brussels'
|
|
},
|
|
'European Economic and Social Committee': {
|
|
'q_number': 'Q641817',
|
|
'label': 'European Economic and Social Committee',
|
|
'relation': 'EU consultative body',
|
|
'viaf': '145822437',
|
|
'coordinates': (50.8467, 4.3772),
|
|
'notes': 'EU consultative body established 1958, represents civil society'
|
|
}
|
|
}
|
|
|
|
print("✍️ Applying manual Wikidata matches...\n")
|
|
|
|
enriched_count = 0
|
|
for inst in be_institutions:
|
|
inst_name = inst['name']
|
|
|
|
if inst_name in manual_matches:
|
|
match = manual_matches[inst_name]
|
|
|
|
print(f" ✅ Applying manual match: {inst_name}")
|
|
print(f" → {match['label']} ({match['q_number']})")
|
|
|
|
# Add Wikidata identifier
|
|
if 'identifiers' not in inst:
|
|
inst['identifiers'] = []
|
|
|
|
# Check if Wikidata already exists
|
|
has_wikidata = any(i.get('identifier_scheme') == 'Wikidata' for i in inst['identifiers'])
|
|
if not has_wikidata:
|
|
inst['identifiers'].append({
|
|
'identifier_scheme': 'Wikidata',
|
|
'identifier_value': match['q_number'],
|
|
'identifier_url': f"https://www.wikidata.org/wiki/{match['q_number']}"
|
|
})
|
|
|
|
# Add VIAF
|
|
has_viaf = any(i.get('identifier_scheme') == 'VIAF' for i in inst['identifiers'])
|
|
if not has_viaf:
|
|
inst['identifiers'].append({
|
|
'identifier_scheme': 'VIAF',
|
|
'identifier_value': match['viaf'],
|
|
'identifier_url': f"https://viaf.org/viaf/{match['viaf']}"
|
|
})
|
|
print(f" 📇 Added VIAF: {match['viaf']}")
|
|
|
|
# Add coordinates
|
|
for location in inst.get('locations', []):
|
|
if location.get('country') == 'BE' and 'latitude' not in location:
|
|
location['latitude'] = match['coordinates'][0]
|
|
location['longitude'] = match['coordinates'][1]
|
|
print(f" 📍 Coordinates: {match['coordinates'][0]}, {match['coordinates'][1]}")
|
|
|
|
# Update description with relationship
|
|
if 'description' in inst:
|
|
inst['description'] = f"{match['relation']} {match['label']}. {inst['description']}"
|
|
else:
|
|
inst['description'] = f"{match['relation']} {match['label']}."
|
|
|
|
# Update provenance
|
|
if 'provenance' not in inst:
|
|
inst['provenance'] = {}
|
|
|
|
# Append enrichment info to extraction_method
|
|
enrichment_note = f"Manual Wikidata enrichment: EU institution linked to {match['label']} ({match['q_number']}). {match['notes']}"
|
|
|
|
if 'extraction_method' in inst['provenance']:
|
|
inst['provenance']['extraction_method'] = f"{inst['provenance']['extraction_method']} + {enrichment_note}"
|
|
else:
|
|
inst['provenance']['extraction_method'] = enrichment_note
|
|
|
|
inst['provenance']['last_updated'] = datetime.now(timezone.utc).isoformat()
|
|
inst['provenance']['wikidata_verified'] = True
|
|
|
|
enriched_count += 1
|
|
print()
|
|
|
|
# Save results (ONLY Belgian institutions)
|
|
output_path = 'data/instances/belgium/be_institutions_enriched_manual.yaml'
|
|
print(f"💾 Saving manual enrichment results to {output_path}...")
|
|
|
|
os.makedirs('data/instances/belgium', exist_ok=True)
|
|
|
|
with open(output_path, 'w', encoding='utf-8') as f:
|
|
yaml.dump(be_institutions, f, allow_unicode=True, sort_keys=False, default_flow_style=False)
|
|
|
|
print(" ✅ Saved\n")
|
|
|
|
# Summary
|
|
total_enriched = sum(1 for inst in be_institutions
|
|
if any(i.get('identifier_scheme') == 'Wikidata' for i in inst.get('identifiers', [])))
|
|
|
|
print("=" * 80)
|
|
print("📊 FINAL BELGIUM ENRICHMENT RESULTS")
|
|
print("=" * 80)
|
|
print(f"Total institutions: {len(be_institutions)}")
|
|
print(f"Wikidata enriched: {total_enriched} ({total_enriched/len(be_institutions)*100:.1f}%)")
|
|
print(f"Still need enrichment: {len(be_institutions) - total_enriched}")
|
|
|
|
if total_enriched >= len(be_institutions) * 0.5:
|
|
print("\n✅ SUCCESS: Achieved 50%+ Wikidata coverage goal!")
|
|
if total_enriched == len(be_institutions):
|
|
print(" 🎯 PERFECT: 100% coverage achieved!")
|
|
|
|
print("\nPhase 1 Belgium: COMPLETE ✅")
|
|
print("\nNext steps:")
|
|
print("1. Merge BE enriched data back into unified dataset")
|
|
print("2. Apply same methodology to United States (US) - 7 institutions")
|
|
print("3. Complete Luxembourg (LU) - 1 institution")
|
|
print("\n")
|
|
|
|
if __name__ == '__main__':
|
|
apply_manual_matches()
|