#!/usr/bin/env python3 """ Extract city names from organization names in PENDING files. This script looks for Dutch city names embedded in organization names and resolves them to proper GHCIDs. Usage: python scripts/resolve_pending_by_city_name.py --dry-run python scripts/resolve_pending_by_city_name.py """ import re import yaml import shutil from pathlib import Path from datetime import datetime, timezone from typing import Optional, Tuple, Dict CUSTODIAN_DIR = Path("/Users/kempersc/apps/glam/data/custodian") # Comprehensive Dutch city list with province and code DUTCH_CITIES = { # Noord-Holland 'amsterdam': ('NH', 'AMS'), 'haarlem': ('NH', 'HAA'), 'alkmaar': ('NH', 'ALK'), 'hilversum': ('NH', 'HIL'), 'zaandam': ('NH', 'ZAA'), 'hoorn': ('NH', 'HOO'), 'enkhuizen': ('NH', 'ENK'), 'purmerend': ('NH', 'PUR'), 'amstelveen': ('NH', 'AMV'), 'heemstede': ('NH', 'HEE'), 'beverwijk': ('NH', 'BEV'), 'velsen': ('NH', 'VEL'), 'castricum': ('NH', 'CAS'), 'huizen': ('NH', 'HUI'), 'bussum': ('NH', 'BUS'), 'naarden': ('NH', 'NAA'), 'weesp': ('NH', 'WEE'), 'edam': ('NH', 'EDA'), 'volendam': ('NH', 'VOL'), 'texel': ('NH', 'TEX'), 'den helder': ('NH', 'DHE'), 'schagen': ('NH', 'SGA'), 'heerhugowaard': ('NH', 'HHW'), 'laren': ('NH', 'LAR'), 'blaricum': ('NH', 'BLA'), 'muiden': ('NH', 'MUI'), 'monnickendam': ('NH', 'MON'), 'bergen': ('NH', 'BER'), 'heiloo': ('NH', 'HEI'), 'uitgeest': ('NH', 'UIT'), 'krommenie': ('NH', 'KRO'), 'wormer': ('NH', 'WOR'), 'westzaan': ('NH', 'WZA'), 'landsmeer': ('NH', 'LAN'), 'broek in waterland': ('NH', 'BIW'), # Zuid-Holland 'rotterdam': ('ZH', 'ROT'), 'den haag': ('ZH', 'DHA'), 'the hague': ('ZH', 'DHA'), "'s-gravenhage": ('ZH', 'DHA'), 'leiden': ('ZH', 'LEI'), 'delft': ('ZH', 'DEL'), 'dordrecht': ('ZH', 'DOR'), 'gouda': ('ZH', 'GOU'), 'schiedam': ('ZH', 'SCH'), 'zoetermeer': ('ZH', 'ZOE'), 'vlaardingen': ('ZH', 'VLA'), 'spijkenisse': ('ZH', 'SPI'), 'alphen aan den rijn': ('ZH', 'AAR'), 'katwijk': ('ZH', 'KAT'), 'leidschendam': ('ZH', 'LSD'), 'voorburg': ('ZH', 'VOO'), 'rijswijk': ('ZH', 'RIJ'), 'wassenaar': ('ZH', 'WAS'), 'oegstgeest': ('ZH', 'OEG'), 'voorschoten': ('ZH', 'VOR'), 'lisse': ('ZH', 'LIS'), 'noordwijk': ('ZH', 'NWK'), 'sassenheim': ('ZH', 'SAS'), 'gorinchem': ('ZH', 'GOR'), 'schoonhoven': ('ZH', 'SOH'), 'woerden': ('ZH', 'WOE'), 'sliedrecht': ('ZH', 'SLI'), 'hoek van holland': ('ZH', 'HVH'), 'maassluis': ('ZH', 'MSL'), 'hellevoetsluis': ('ZH', 'HEV'), 'brielle': ('ZH', 'BRI'), 'middelharnis': ('ZH', 'MID'), 'goedereede': ('ZH', 'GOE'), 'papendrecht': ('ZH', 'PAP'), 'hendrik-ido-ambacht': ('ZH', 'HIA'), 'zwijndrecht': ('ZH', 'ZWI'), 'barendrecht': ('ZH', 'BAR'), 'ridderkerk': ('ZH', 'RID'), 'capelle aan den ijssel': ('ZH', 'CAP'), # Utrecht 'utrecht': ('UT', 'UTR'), 'amersfoort': ('UT', 'AME'), 'zeist': ('UT', 'ZEI'), 'nieuwegein': ('UT', 'NIE'), 'veenendaal': ('UT', 'VEE'), 'houten': ('UT', 'HOU'), 'soest': ('UT', 'SOE'), 'baarn': ('UT', 'BAA'), 'bunnik': ('UT', 'BUN'), 'maarssen': ('UT', 'MAA'), 'de bilt': ('UT', 'BIL'), 'bilthoven': ('UT', 'BIT'), 'doorn': ('UT', 'DOO'), 'driebergen': ('UT', 'DRI'), 'wijk bij duurstede': ('UT', 'WBD'), 'breukelen': ('UT', 'BRE'), 'loenen': ('UT', 'LOE'), 'oudewater': ('UT', 'OUD'), 'rhenen': ('UT', 'RHE'), 'vianen': ('UT', 'VIA'), 'culemborg': ('UT', 'CUL'), # Gelderland 'arnhem': ('GE', 'ARN'), 'nijmegen': ('GE', 'NIJ'), 'apeldoorn': ('GE', 'APE'), 'ede': ('GE', 'EDE'), 'wageningen': ('GE', 'WAG'), 'doetinchem': ('GE', 'DOE'), 'zutphen': ('GE', 'ZUT'), 'harderwijk': ('GE', 'HAR'), 'zevenaar': ('GE', 'ZEV'), 'tiel': ('GE', 'TIE'), 'barneveld': ('GE', 'BNV'), 'elburg': ('GE', 'ELB'), 'nunspeet': ('GE', 'NUN'), 'putten': ('GE', 'PUT'), 'ermelo': ('GE', 'ERM'), 'epe': ('GE', 'EPE'), 'hattem': ('GE', 'HAT'), 'zaltbommel': ('GE', 'ZAL'), 'winterswijk': ('GE', 'WIN'), 'oosterbeek': ('GE', 'OOS'), 'velp': ('GE', 'VEL'), 'buren': ('GE', 'BUR'), 'groenlo': ('GE', 'GRO'), 'borculo': ('GE', 'BOR'), 'lochem': ('GE', 'LOC'), 'hengelo': ('GE', 'HEN'), 'bronkhorst': ('GE', 'BRO'), 'elst': ('GE', 'ELS'), 'bemmel': ('GE', 'BEM'), 'duiven': ('GE', 'DUI'), 'westervoort': ('GE', 'WES'), 'renkum': ('GE', 'REN'), 'rozendaal': ('GE', 'ROZ'), 'berkelland': ('GE', 'BKL'), 'aalten': ('GE', 'AAL'), 'lichtenvoorde': ('GE', 'LIC'), # Noord-Brabant 'eindhoven': ('NB', 'EIN'), 'tilburg': ('NB', 'TIL'), 'breda': ('NB', 'BRE'), "'s-hertogenbosch": ('NB', 'DBO'), 'den bosch': ('NB', 'DBO'), 'helmond': ('NB', 'HEL'), 'oss': ('NB', 'OSS'), 'roosendaal': ('NB', 'ROS'), 'bergen op zoom': ('NB', 'BOZ'), 'waalwijk': ('NB', 'WAA'), 'uden': ('NB', 'UDE'), 'veghel': ('NB', 'VEG'), 'boxtel': ('NB', 'BOX'), 'oisterwijk': ('NB', 'OIS'), 'vught': ('NB', 'VUG'), 'cuijk': ('NB', 'CUI'), 'deurne': ('NB', 'DEU'), 'geldrop': ('NB', 'GEL'), 'mierlo': ('NB', 'MIE'), 'nuenen': ('NB', 'NUE'), 'valkenswaard': ('NB', 'VAL'), 'heeze': ('NB', 'HEE'), 'best': ('NB', 'BES'), 'son': ('NB', 'SON'), 'etten-leur': ('NB', 'ETL'), 'oosterhout': ('NB', 'OOH'), 'dongen': ('NB', 'DON'), 'gilze': ('NB', 'GIL'), 'rijen': ('NB', 'RIJ'), 'made': ('NB', 'MAD'), 'raamsdonk': ('NB', 'RAA'), 'geertruidenberg': ('NB', 'GEE'), 'heusden': ('NB', 'HEU'), 'drunen': ('NB', 'DRU'), 'vlijmen': ('NB', 'VLI'), # Limburg 'maastricht': ('LI', 'MAA'), 'venlo': ('LI', 'VEN'), 'heerlen': ('LI', 'HEE'), 'roermond': ('LI', 'ROE'), 'sittard': ('LI', 'SIT'), 'geleen': ('LI', 'GEL'), 'weert': ('LI', 'WEE'), 'kerkrade': ('LI', 'KER'), 'valkenburg': ('LI', 'VAL'), 'vaals': ('LI', 'VAA'), 'meerssen': ('LI', 'MEE'), 'brunssum': ('LI', 'BRU'), 'landgraaf': ('LI', 'LAN'), 'stein': ('LI', 'STE'), 'beek': ('LI', 'BEE'), 'eijsden': ('LI', 'EIJ'), 'gulpen': ('LI', 'GUL'), 'margraten': ('LI', 'MAR'), 'simpelveld': ('LI', 'SIM'), 'thorn': ('LI', 'THO'), 'horst': ('LI', 'HOR'), 'venray': ('LI', 'VRY'), 'tegelen': ('LI', 'TEG'), 'blerick': ('LI', 'BLE'), 'arcen': ('LI', 'ARC'), 'neeritter': ('LI', 'NEE'), # Overijssel 'zwolle': ('OV', 'ZWO'), 'deventer': ('OV', 'DEV'), 'enschede': ('OV', 'ENS'), 'hengelo': ('OV', 'HEN'), 'almelo': ('OV', 'ALM'), 'kampen': ('OV', 'KAM'), 'oldenzaal': ('OV', 'OLD'), 'hardenberg': ('OV', 'HDB'), 'rijssen': ('OV', 'RIJ'), 'holten': ('OV', 'HOL'), 'raalte': ('OV', 'RAA'), 'ommen': ('OV', 'OMM'), 'dalfsen': ('OV', 'DAL'), 'staphorst': ('OV', 'STA'), 'giethoorn': ('OV', 'GIE'), 'hasselt': ('OV', 'HAS'), 'steenwijk': ('OV', 'STW'), 'vollenhove': ('OV', 'VOL'), 'blokzijl': ('OV', 'BLO'), 'tubbergen': ('OV', 'TUB'), 'losser': ('OV', 'LOS'), 'denekamp': ('OV', 'DEN'), 'haaksbergen': ('OV', 'HAA'), 'markelo': ('OV', 'MRK'), 'diepenheim': ('OV', 'DIE'), 'goor': ('OV', 'GOO'), 'vriezenveen': ('OV', 'VRI'), 'wierden': ('OV', 'WIE'), 'nijverdal': ('OV', 'NIJ'), 'hellendoorn': ('OV', 'HEL'), # Friesland 'leeuwarden': ('FR', 'LEE'), 'drachten': ('FR', 'DRA'), 'sneek': ('FR', 'SNE'), 'heerenveen': ('FR', 'HEE'), 'harlingen': ('FR', 'HAR'), 'dokkum': ('FR', 'DOK'), 'bolsward': ('FR', 'BOL'), 'franeker': ('FR', 'FRA'), 'joure': ('FR', 'JOU'), 'lemmer': ('FR', 'LEM'), 'workum': ('FR', 'WOR'), 'makkum': ('FR', 'MAK'), 'hindeloopen': ('FR', 'HIN'), 'stavoren': ('FR', 'STA'), 'sloten': ('FR', 'SLO'), 'koudum': ('FR', 'KOU'), 'balk': ('FR', 'BAL'), 'gorredijk': ('FR', 'GOR'), 'wolvega': ('FR', 'WOL'), 'akkrum': ('FR', 'AKK'), 'burgum': ('FR', 'BUR'), 'kollum': ('FR', 'KOL'), 'damwoude': ('FR', 'DAM'), 'ferwerd': ('FR', 'FER'), # Drenthe 'assen': ('DR', 'ASS'), 'emmen': ('DR', 'EMM'), 'hoogeveen': ('DR', 'HOO'), 'meppel': ('DR', 'MEP'), 'coevorden': ('DR', 'COE'), 'borger': ('DR', 'BOR'), 'dwingeloo': ('DR', 'DWI'), 'westerbork': ('DR', 'WES'), 'beilen': ('DR', 'BEI'), 'roden': ('DR', 'ROD'), 'norg': ('DR', 'NOR'), 'zuidlaren': ('DR', 'ZUI'), 'gieten': ('DR', 'GIE'), 'exloo': ('DR', 'EXL'), 'orvelte': ('DR', 'ORV'), 'veenhuizen': ('DR', 'VHU'), # Groningen 'groningen': ('GR', 'GRO'), 'veendam': ('GR', 'VEE'), 'winschoten': ('GR', 'WIN'), 'hoogezand': ('GR', 'HZD'), 'stadskanaal': ('GR', 'STK'), 'delfzijl': ('GR', 'DEL'), 'appingedam': ('GR', 'APP'), 'ter apel': ('GR', 'TAP'), 'leek': ('GR', 'LEE'), 'marum': ('GR', 'MAR'), 'zuidhorn': ('GR', 'ZDH'), 'uithuizen': ('GR', 'UIT'), 'loppersum': ('GR', 'LOP'), 'middelstum': ('GR', 'MID'), 'baflo': ('GR', 'BAF'), 'usquert': ('GR', 'USQ'), 'warffum': ('GR', 'WAR'), # Zeeland 'middelburg': ('ZE', 'MID'), 'vlissingen': ('ZE', 'VLI'), 'goes': ('ZE', 'GOE'), 'terneuzen': ('ZE', 'TER'), 'hulst': ('ZE', 'HUL'), 'zierikzee': ('ZE', 'ZIE'), 'veere': ('ZE', 'VEE'), 'domburg': ('ZE', 'DOM'), 'sluis': ('ZE', 'SLU'), 'yerseke': ('ZE', 'YER'), 'tholen': ('ZE', 'THO'), 'sint-maartensdijk': ('ZE', 'SMD'), 'bruinisse': ('ZE', 'BRU'), 'brouwershaven': ('ZE', 'BRO'), # Flevoland 'almere': ('FL', 'ALM'), 'lelystad': ('FL', 'LEL'), 'dronten': ('FL', 'DRO'), 'emmeloord': ('FL', 'EMM'), 'urk': ('FL', 'URK'), 'zeewolde': ('FL', 'ZEE'), 'biddinghuizen': ('FL', 'BID'), } # Special place names that aren't cities but indicate locations SPECIAL_LOCATIONS = { 'muiderslot': ('NH', 'MUI'), # Muiden castle 'hilde': ('NH', 'CAS'), # Huis van Hilde in Castricum 'veenhuizen': ('DR', 'VHU'), 'kinderdijk': ('ZH', 'KIN'), 'zaanstreek': ('NH', 'ZAA'), 'tweestromenland': ('GE', 'MAA'), # Maas-Waal region } def extract_city_from_name(org_name: str) -> Optional[Tuple[str, str, str]]: """ Extract city name from organization name. Returns (province_code, city_code, city_name) or None. """ name_lower = org_name.lower() # First check special locations for location, (prov, code) in SPECIAL_LOCATIONS.items(): if location in name_lower: return (prov, code, location) # Sort cities by length (longest first) to match "hoek van holland" before "holland" sorted_cities = sorted(DUTCH_CITIES.keys(), key=len, reverse=True) for city in sorted_cities: # Check if city name appears as a word boundary pattern = r'\b' + re.escape(city) + r'\b' if re.search(pattern, name_lower): prov, code = DUTCH_CITIES[city] return (prov, code, city) return None def extract_abbreviation(name: str) -> str: """Extract abbreviation from organization name.""" skip_words = { 'de', 'het', 'een', 'van', 'voor', 'in', 'op', 'te', 'den', 'der', 'des', 'en', 'of', 'the', 'a', 'an', 'and', 'or', 'museum', 'stichting', } name_clean = re.sub(r'[^\w\s]', ' ', name) words = [w for w in name_clean.split() if w.lower() not in skip_words and len(w) > 1] if not words: words = name_clean.split()[:3] if len(words) == 1: abbrev = words[0][:4].upper() else: abbrev = ''.join(w[0] for w in words[:5]).upper() return abbrev if abbrev else 'XXX' def get_institution_type(data: Dict) -> str: """Get institution type code from data.""" type_map = { 'MUSEUM': 'M', 'ARCHIVE': 'A', 'LIBRARY': 'L', 'GALLERY': 'G', 'RESEARCH_CENTER': 'R', 'EDUCATION_PROVIDER': 'E', 'OFFICIAL_INSTITUTION': 'O', 'COLLECTING_SOCIETY': 'S', 'HOLY_SITES': 'H', 'DIGITAL_PLATFORM': 'D', 'BOTANICAL_ZOO': 'B', 'CORPORATION': 'C', 'FEATURES': 'F', 'INTANGIBLE_HERITAGE_GROUP': 'I', 'MIXED': 'X', 'PERSONAL_COLLECTION': 'P', 'NGO': 'N', 'TASTE_SMELL': 'T', 'UNKNOWN': 'U', } inst_type = data.get('institution_type', 'MUSEUM') return type_map.get(inst_type, 'M') def process_pending_file(filepath: Path, dry_run: bool = True) -> Optional[str]: """Process a single PENDING file.""" try: with open(filepath, 'r', encoding='utf-8') as f: data = yaml.safe_load(f) emic_name = data.get('custodian_name', {}).get('emic_name', '') if not emic_name: return None # Try to extract city from name result = extract_city_from_name(emic_name) if not result: return None prov, city_code, city_name = result inst_type = get_institution_type(data) abbrev = extract_abbreviation(emic_name) # Generate new GHCID new_ghcid = f"NL-{prov}-{city_code.upper()}-{inst_type}-{abbrev}" # Check for collision new_path = CUSTODIAN_DIR / f"{new_ghcid}.yaml" if new_path.exists() and new_path != filepath: # Add name suffix for collision name_slug = re.sub(r'[^\w]+', '-', emic_name.lower()).strip('-')[:30] new_ghcid = f"{new_ghcid}-{name_slug}" new_path = CUSTODIAN_DIR / f"{new_ghcid}.yaml" if dry_run: print(f"[WOULD RESOLVE] {emic_name}") print(f" City found: {city_name}") print(f" -> {new_ghcid}.yaml") return 'dry_run' # Update data data['ghcid_current'] = new_ghcid if 'location' not in data: data['location'] = {} data['location']['city'] = city_name.title() data['location']['country'] = 'NL' # Add resolution provenance if 'ghcid_resolution' not in data: data['ghcid_resolution'] = {} data['ghcid_resolution']['method'] = 'city_name_extraction' data['ghcid_resolution']['resolved_at'] = datetime.now(timezone.utc).isoformat() data['ghcid_resolution']['extracted_city'] = city_name # Write and rename with open(filepath, 'w', encoding='utf-8') as f: yaml.dump(data, f, default_flow_style=False, allow_unicode=True, sort_keys=False) shutil.move(filepath, new_path) print(f"[RESOLVED] {emic_name}") print(f" City: {city_name} -> {new_ghcid}.yaml") return new_ghcid except Exception as e: print(f"[ERROR] {filepath.name}: {e}") return None def main(): import argparse parser = argparse.ArgumentParser() parser.add_argument('--dry-run', action='store_true') parser.add_argument('--limit', type=int, default=0) args = parser.parse_args() # Find NL PENDING files only pending_files = list(CUSTODIAN_DIR.glob("NL-XX-XXX-PENDING-*.yaml")) print(f"Found {len(pending_files)} NL PENDING files") if args.limit: pending_files = pending_files[:args.limit] resolved = 0 failed = 0 for filepath in pending_files: result = process_pending_file(filepath, dry_run=args.dry_run) if result: resolved += 1 else: failed += 1 print(f"\n{'Would resolve' if args.dry_run else 'Resolved'}: {resolved}") print(f"No city found: {failed}") if __name__ == '__main__': main()